X64: Convert doubles to int32 inline (without calling runtime).

src/x64/codegen-x64.cc

Index: src/x64/codegen-x64.cc
diff --git a/src/x64/codegen-x64.cc b/src/x64/codegen-x64.cc
index 8e6dbef2d16186720abee8dc4b0ea81299b48c8a..2e26662e5dcd5fe2fa449a0c8eadc61892d3501c 100644
--- a/src/x64/codegen-x64.cc
+++ b/src/x64/codegen-x64.cc
@@ -225,14 +225,19 @@ class FloatingPointHelper : public AllStatic {
                                 Register lhs,
                                 Register rhs);
 
-  // Code pattern for loading a floating point value and converting it
-  // to a 32 bit integer. Input value must be either a smi or a heap number
-  // object.
-  // Returns operands as 32-bit sign extended integers in a general purpose
-  // registers.
-  static void LoadInt32Operand(MacroAssembler* masm,
-                               const Operand& src,
-                               Register dst);
+  // Code pattern for loading a floating point argument and converting it
+  // to a 32 bit integer by truncating. Input value must be either a smi or
+  // a heap number object, otherwise the on_non_number label is jumped to.
+  // Expects arguments in register src, and returns result in register dst.
+  // Uses rcx and rax (src may not be either of these).
+  static void LoadInt32OperandSSE3(MacroAssembler* masm,
+                                   Register dst,
+                                   Register src,
+                                   Label* on_non_number);
+  static void LoadInt32OperandFPU(MacroAssembler* masm,
+                                  Register dst,
+                                  Register src,
+                                  Label* on_non_number);
 
   // Test if operands are smi or number objects (fp). Requirements:
   // operand_1 in rax, operand_2 in rdx; falls through on float or smi
@@ -7254,15 +7259,103 @@ void FloatingPointHelper::LoadFloatOperands(MacroAssembler* masm,
 }
 
 
-void FloatingPointHelper::LoadInt32Operand(MacroAssembler* masm,
-                                           const Operand& src,
-                                           Register dst) {
-  // TODO(X64): Convert number operands to int32 values.
-  // Don't convert a Smi to a double first.
-  UNIMPLEMENTED();
+void FloatingPointHelper::LoadInt32OperandSSE3(MacroAssembler* masm,
+                                               Register dst,
+                                               Register src,
+                                               Label* on_non_number) {
+  ASSERT(!src.is(rax));
+  ASSERT(!src.is(rcx));
+
+  CpuFeatures::Scope scope(CpuFeatures::SSE3);
+
+  Label non_smi;
+  Label done;
+  Label non_smi_value;
+
+  __ JumpIfNotSmi(src, &non_smi);
+  __ SmiToInteger32(dst, src);
+  __ jmp(&done);
+
+  // Do a full conversion by operating on the double bits.
+  __ bind(&non_smi_value);
+  __ addq(rsp, Immediate(kPointerSize));
+  __ fnclex();
+  __ movq(src, FieldOperand(src, HeapNumber::kValueOffset));
+  __ DoubleToInteger32(dst, src, rax);
+  __ jmp(&done);
+
+  __ bind(&non_smi);
+  // Check that we have a HeapNumber.
+  __ CompareRoot(FieldOperand(src, HeapObject::kMapOffset),
+                 Heap::kHeapNumberMapRootIndex);
+  __ j(not_equal, on_non_number);
+
+  // Try quick conversion, if the double holds a valid smi value.
+
+  __ fld_d(FieldOperand(src, HeapNumber::kValueOffset));
+  __ subq(rsp, Immediate(kPointerSize));
+  __ fisttp_s(Operand(rsp, 0 * kPointerSize));
+  __ fnstsw_ax();
+  __ testl(rax, Immediate(1));
+  __ j(not_zero, &non_smi_value);
+  __ pop(dst);
+
+  __ bind(&done);
+}
+
+
+void FloatingPointHelper::LoadInt32OperandFPU(MacroAssembler* masm,
+                                               Register dst,
+                                               Register src,
+                                               Label* on_non_number) {
+  ASSERT(!src.is(rax));
+  ASSERT(!src.is(rcx));
+
+  Label non_smi;
+  Label done;
+  Label non_smi_value;
+
+  __ JumpIfNotSmi(src, &non_smi);
+  __ SmiToInteger32(dst, src);
+  __ jmp(&done);
+
+  // Do a full conversion by operating on the double bits.
+  __ bind(&non_smi_value);
+  __ addq(rsp, Immediate(kPointerSize));
+  __ movq(src, FieldOperand(src, HeapNumber::kValueOffset));
+  __ DoubleToInteger32(dst, src, rax);
+  __ jmp(&done);
+
+  __ bind(&non_smi);
+  // Check that we have a HeapNumber.
+  __ CompareRoot(FieldOperand(src, HeapObject::kMapOffset),
+                 Heap::kHeapNumberMapRootIndex);
+  __ j(not_equal, on_non_number);
+
+  // Try quick conversion, if the double holds a valid smi value.
+
+  __ fld_d(FieldOperand(src, HeapNumber::kValueOffset));
+  __ subq(rsp, Immediate(kPointerSize));
+  __ fist_s(Operand(rsp, 0 * kPointerSize));
+  __ fild_s(Operand(rsp, 0 * kPointerSize));
+  __ fucompp();
+  __ fnstsw_ax();
+  if (CpuFeatures::IsSupported(CpuFeatures::SAHF)) {
+    __ sahf();
+    __ j(not_zero, &non_smi_value);
+    __ j(parity_even, &non_smi_value);
+  } else {
+    __ and_(rax, Immediate(0x4400));
+    __ cmpl(rax, Immediate(0x4000));
+    __ j(not_equal, &non_smi_value);
+  }
+  __ pop(dst);
+
+  __ bind(&done);
 }
 
 
+
 void FloatingPointHelper::LoadFloatOperands(MacroAssembler* masm) {
   Label load_smi_1, load_smi_2, done_load_1, done;
   __ movq(kScratchRegister, Operand(rsp, 2 * kPointerSize));
@@ -7402,8 +7495,6 @@ void GenericBinaryOpStub::GenerateSmiCode(MacroAssembler* masm, Label* slow) {
     case Token::SHL:
     case Token::SHR:
     case Token::SAR:
-      // Move the second operand into register ecx.
-      __ movl(rcx, rbx);
       // Perform the operation.
       switch (op_) {
         case Token::SAR:
@@ -7499,86 +7590,58 @@ void GenericBinaryOpStub::Generate(MacroAssembler* masm) {
     case Token::SAR:
     case Token::SHL:
     case Token::SHR: {
-      FloatingPointHelper::CheckFloatOperands(masm, &call_runtime);
-      // TODO(X64): Don't convert a Smi to float and then back to int32
-      // afterwards.
-      FloatingPointHelper::LoadFloatOperands(masm);
-
       Label skip_allocation, non_smi_result, operand_conversion_failure;

[Lasse Reichstein, 2009/10/07 16:54:24]

I'm considering using CheckFloatOperands first, to avoid converting one operand
and then going to the runtime if the other is a non-number. It will double the
smi checks, though.

-
-      // Reserve space for converted numbers.
-      __ subq(rsp, Immediate(2 * kPointerSize));
-
+      // Arguments in rax/rsp[1] and rdx/rsp[0].
+      // Move rhs to rbx, since conversion uses rax to read FPU status word.
+      __ movq(rbx, rax);
       if (use_sse3_) {
-        // Truncate the operands to 32-bit integers and check for
-        // exceptions in doing so.
-        CpuFeatures::Scope scope(CpuFeatures::SSE3);
-        __ fisttp_s(Operand(rsp, 0 * kPointerSize));
-        __ fisttp_s(Operand(rsp, 1 * kPointerSize));
-        __ fnstsw_ax();
-        __ testl(rax, Immediate(1));
-        __ j(not_zero, &operand_conversion_failure);
+        FloatingPointHelper::LoadInt32OperandSSE3(masm,
+                                                  rdx,
+                                                  rdx,
+                                                  &operand_conversion_failure);
+        FloatingPointHelper::LoadInt32OperandSSE3(masm,
+                                                  rcx,
+                                                  rbx,
+                                                  &operand_conversion_failure);
       } else {
-        // Check if right operand is int32.
-        __ fist_s(Operand(rsp, 0 * kPointerSize));
-        __ fild_s(Operand(rsp, 0 * kPointerSize));
-        __ fucompp();
-        __ fnstsw_ax();
-        if (CpuFeatures::IsSupported(CpuFeatures::SAHF)) {
-          __ sahf();
-          __ j(not_zero, &operand_conversion_failure);
-          __ j(parity_even, &operand_conversion_failure);
-        } else {
-          __ and_(rax, Immediate(0x4400));
-          __ cmpl(rax, Immediate(0x4000));
-          __ j(not_zero, &operand_conversion_failure);
-        }
-        // Check if left operand is int32.
-        __ fist_s(Operand(rsp, 1 * kPointerSize));
-        __ fild_s(Operand(rsp, 1 * kPointerSize));
-        __ fucompp();
-        __ fnstsw_ax();
-        if (CpuFeatures::IsSupported(CpuFeatures::SAHF)) {
-          __ sahf();
-          __ j(not_zero, &operand_conversion_failure);
-          __ j(parity_even, &operand_conversion_failure);
-        } else {
-          __ and_(rax, Immediate(0x4400));
-          __ cmpl(rax, Immediate(0x4000));
-          __ j(not_zero, &operand_conversion_failure);
-        }
+        FloatingPointHelper::LoadInt32OperandFPU(masm,
+                                                 rdx,
+                                                 rdx,
+                                                 &operand_conversion_failure);
+        FloatingPointHelper::LoadInt32OperandFPU(masm,
+                                                 rcx,
+                                                 rbx,
+                                                 &operand_conversion_failure);
       }
+      // 32-bit values in rbx, rcx.
 
-      // Get int32 operands and perform bitop.
-      __ pop(rcx);
-      __ pop(rax);
       switch (op_) {
-        case Token::BIT_OR:  __ or_(rax, rcx); break;
-        case Token::BIT_AND: __ and_(rax, rcx); break;
-        case Token::BIT_XOR: __ xor_(rax, rcx); break;
-        case Token::SAR: __ sarl(rax); break;
-        case Token::SHL: __ shll(rax); break;
-        case Token::SHR: __ shrl(rax); break;
+        case Token::BIT_OR:  __ orl(rdx, rcx); break;
+        case Token::BIT_AND: __ andl(rdx, rcx); break;
+        case Token::BIT_XOR: __ xorl(rdx, rcx); break;
+        case Token::SAR: __ sarl(rdx); break;
+        case Token::SHL: __ shll(rdx); break;
+        case Token::SHR: __ shrl(rdx); break;
         default: UNREACHABLE();
       }
       if (op_ == Token::SHR) {
         // Check if result is non-negative and fits in a smi.
-        __ testl(rax, Immediate(0xc0000000));
+        __ testl(rdx, Immediate(0xc0000000));
         __ j(not_zero, &non_smi_result);
       } else {
         // Check if result fits in a smi.
-        __ cmpl(rax, Immediate(0xc0000000));
+        __ cmpl(rdx, Immediate(0xc0000000));
         __ j(negative, &non_smi_result);
       }
       // Tag smi result and return.
-      __ Integer32ToSmi(rax, rax);
+      __ Integer32ToSmi(rax, rdx);
       __ ret(2 * kPointerSize);
 
       // All ops except SHR return a signed int32 that we load in a HeapNumber.
       if (op_ != Token::SHR) {
         __ bind(&non_smi_result);
         // Allocate a heap number if needed.
-        __ movsxlq(rbx, rax);  // rbx: sign extended 32-bit result
+        __ movsxlq(rbx, rdx);  // rbx: sign extended 32-bit result
         switch (mode_) {
           case OVERWRITE_LEFT:
           case OVERWRITE_RIGHT:
@@ -7602,21 +7665,8 @@ void GenericBinaryOpStub::Generate(MacroAssembler* masm) {
         __ ret(2 * kPointerSize);
       }
 
-      // Clear the FPU exception flag and reset the stack before calling
-      // the runtime system.
+      // Fast-case operation failed, so call the runtime to perform the operation.
       __ bind(&operand_conversion_failure);
-      __ addq(rsp, Immediate(2 * kPointerSize));
-      if (use_sse3_) {
-        // If we've used the SSE3 instructions for truncating the
-        // floating point values to integers and it failed, we have a
-        // pending #IA exception. Clear it.
-        __ fnclex();
-      } else {
-        // The non-SSE3 variant does early bailout if the right
-        // operand isn't a 32-bit integer, so we may have a single
-        // value on the FPU stack we need to get rid of.
-        __ ffree(0);
-      }
 
       // SHR should return uint32 - go to runtime for non-smi/negative result.
       if (op_ == Token::SHR) {