X64: Convert doubles to int32 inline (without calling runtime).

src/x64/codegen-x64.cc

 // Copyright 2009 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 207 matching lines @@
   // Input values must be either smi or heap number objects (fp values).
   // Requirements:
   // Register version: operands in registers lhs and rhs.
   // Stack version: operands on TOS+1 and TOS+2.
   // Returns operands as floating point numbers on fp stack.
   static void LoadFloatOperands(MacroAssembler* masm);
   static void LoadFloatOperands(MacroAssembler* masm,
                                 Register lhs,
                                 Register rhs);
 
-  // Code pattern for loading a floating point value and converting it
-  // to a 32 bit integer. Input value must be either a smi or a heap number
-  // object.
-  // Returns operands as 32-bit sign extended integers in a general purpose
-  // registers.
-  static void LoadInt32Operand(MacroAssembler* masm,
-                               const Operand& src,
-                               Register dst);
+  // Code pattern for loading a floating point argument and converting it
+  // to a 32 bit integer by truncating. Input value must be either a smi or
+  // a heap number object, otherwise the on_non_number label is jumped to.
+  // Expects arguments in register src, and returns result in register dst.
+  // Uses rcx and rax (src may not be either of these).
+  static void LoadInt32OperandSSE3(MacroAssembler* masm,
+                                   Register dst,
+                                   Register src,
+                                   Label* on_non_number);
+  static void LoadInt32OperandFPU(MacroAssembler* masm,
+                                  Register dst,
+                                  Register src,
+                                  Label* on_non_number);
 
   // Test if operands are smi or number objects (fp). Requirements:
   // operand_1 in rax, operand_2 in rdx; falls through on float or smi
   // operands, jumps to the non_float label otherwise.
   static void CheckFloatOperands(MacroAssembler* masm,
                                  Label* non_float);
 
   // Allocate a heap number in new space with undefined value.
   // Returns tagged pointer in result, or jumps to need_gc if new space is full.
   static void AllocateHeapNumber(MacroAssembler* masm,
@@ skipping 7001 matching lines @@
 void FloatingPointHelper::LoadFloatOperands(MacroAssembler* masm,
                                             XMMRegister dst1,
                                             XMMRegister dst2) {
   __ movq(kScratchRegister, Operand(rsp, 2 * kPointerSize));
   LoadFloatOperand(masm, kScratchRegister, dst1);
   __ movq(kScratchRegister, Operand(rsp, 1 * kPointerSize));
   LoadFloatOperand(masm, kScratchRegister, dst2);
 }
 
 
-void FloatingPointHelper::LoadInt32Operand(MacroAssembler* masm,
-                                           const Operand& src,
-                                           Register dst) {
-  // TODO(X64): Convert number operands to int32 values.
-  // Don't convert a Smi to a double first.
-  UNIMPLEMENTED();
+void FloatingPointHelper::LoadInt32OperandSSE3(MacroAssembler* masm,
+                                               Register dst,
+                                               Register src,
+                                               Label* on_non_number) {
+  ASSERT(!src.is(rax));
+  ASSERT(!src.is(rcx));
+
+  CpuFeatures::Scope scope(CpuFeatures::SSE3);
+
+  Label non_smi;
+  Label done;
+  Label non_smi_value;
+
+  __ JumpIfNotSmi(src, &non_smi);
+  __ SmiToInteger32(dst, src);
+  __ jmp(&done);
+
+  // Do a full conversion by operating on the double bits.
+  __ bind(&non_smi_value);
+  __ addq(rsp, Immediate(kPointerSize));
+  __ fnclex();
+  __ movq(src, FieldOperand(src, HeapNumber::kValueOffset));
+  __ DoubleToInteger32(dst, src, rax);
+  __ jmp(&done);
+
+  __ bind(&non_smi);
+  // Check that we have a HeapNumber.
+  __ CompareRoot(FieldOperand(src, HeapObject::kMapOffset),
+                 Heap::kHeapNumberMapRootIndex);
+  __ j(not_equal, on_non_number);
+
+  // Try quick conversion, if the double holds a valid smi value.
+
+  __ fld_d(FieldOperand(src, HeapNumber::kValueOffset));
+  __ subq(rsp, Immediate(kPointerSize));
+  __ fisttp_s(Operand(rsp, 0 * kPointerSize));
+  __ fnstsw_ax();
+  __ testl(rax, Immediate(1));
+  __ j(not_zero, &non_smi_value);
+  __ pop(dst);
+
+  __ bind(&done);
 }
 
 
+void FloatingPointHelper::LoadInt32OperandFPU(MacroAssembler* masm,
+                                               Register dst,
+                                               Register src,
+                                               Label* on_non_number) {
+  ASSERT(!src.is(rax));
+  ASSERT(!src.is(rcx));
+
+  Label non_smi;
+  Label done;
+  Label non_smi_value;
+
+  __ JumpIfNotSmi(src, &non_smi);
+  __ SmiToInteger32(dst, src);
+  __ jmp(&done);
+
+  // Do a full conversion by operating on the double bits.
+  __ bind(&non_smi_value);
+  __ addq(rsp, Immediate(kPointerSize));
+  __ movq(src, FieldOperand(src, HeapNumber::kValueOffset));
+  __ DoubleToInteger32(dst, src, rax);
+  __ jmp(&done);
+
+  __ bind(&non_smi);
+  // Check that we have a HeapNumber.
+  __ CompareRoot(FieldOperand(src, HeapObject::kMapOffset),
+                 Heap::kHeapNumberMapRootIndex);
+  __ j(not_equal, on_non_number);
+
+  // Try quick conversion, if the double holds a valid smi value.
+
+  __ fld_d(FieldOperand(src, HeapNumber::kValueOffset));
+  __ subq(rsp, Immediate(kPointerSize));
+  __ fist_s(Operand(rsp, 0 * kPointerSize));
+  __ fild_s(Operand(rsp, 0 * kPointerSize));
+  __ fucompp();
+  __ fnstsw_ax();
+  if (CpuFeatures::IsSupported(CpuFeatures::SAHF)) {
+    __ sahf();
+    __ j(not_zero, &non_smi_value);
+    __ j(parity_even, &non_smi_value);
+  } else {
+    __ and_(rax, Immediate(0x4400));
+    __ cmpl(rax, Immediate(0x4000));
+    __ j(not_equal, &non_smi_value);
+  }
+  __ pop(dst);
+
+  __ bind(&done);
+}
+
+
+
 void FloatingPointHelper::LoadFloatOperands(MacroAssembler* masm) {
   Label load_smi_1, load_smi_2, done_load_1, done;
   __ movq(kScratchRegister, Operand(rsp, 2 * kPointerSize));
   __ JumpIfSmi(kScratchRegister, &load_smi_1);
   __ fld_d(FieldOperand(kScratchRegister, HeapNumber::kValueOffset));
   __ bind(&done_load_1);
 
   __ movq(kScratchRegister, Operand(rsp, 1 * kPointerSize));
   __ JumpIfSmi(kScratchRegister, &load_smi_2);
   __ fld_d(FieldOperand(kScratchRegister, HeapNumber::kValueOffset));
@@ skipping 119 matching lines @@
       __ SmiAnd(rax, rax, rbx);
       break;
 
     case Token::BIT_XOR:
       __ SmiXor(rax, rax, rbx);
       break;
 
     case Token::SHL:
     case Token::SHR:
     case Token::SAR:
-      // Move the second operand into register ecx.
-      __ movl(rcx, rbx);
       // Perform the operation.
       switch (op_) {
         case Token::SAR:
           __ SmiShiftArithmeticRight(rax, rax, rbx);
           break;
         case Token::SHR:
           __ SmiShiftLogicalRight(rax, rax, rbx, slow);
           break;
         case Token::SHL:
           __ SmiShiftLeft(rax, rax, rbx, slow);
@@ skipping 75 matching lines @@
     case Token::MOD: {
       // For MOD we go directly to runtime in the non-smi case.
       break;
     }
     case Token::BIT_OR:
     case Token::BIT_AND:
     case Token::BIT_XOR:
     case Token::SAR:
     case Token::SHL:
     case Token::SHR: {
-      FloatingPointHelper::CheckFloatOperands(masm, &call_runtime);
-      // TODO(X64): Don't convert a Smi to float and then back to int32
-      // afterwards.
-      FloatingPointHelper::LoadFloatOperands(masm);
+      Label skip_allocation, non_smi_result, operand_conversion_failure;

[Lasse Reichstein, 2009/10/07 16:54:24]

I'm considering using CheckFloatOperands first, to avoid converting one operand
and then going to the runtime if the other is a non-number. It will double the
smi checks, though.

+      // Arguments in rax/rsp[1] and rdx/rsp[0].
+      // Move rhs to rbx, since conversion uses rax to read FPU status word.
+      __ movq(rbx, rax);
+      if (use_sse3_) {
+        FloatingPointHelper::LoadInt32OperandSSE3(masm,
+                                                  rdx,
+                                                  rdx,
+                                                  &operand_conversion_failure);
+        FloatingPointHelper::LoadInt32OperandSSE3(masm,
+                                                  rcx,
+                                                  rbx,
+                                                  &operand_conversion_failure);
+      } else {
+        FloatingPointHelper::LoadInt32OperandFPU(masm,
+                                                 rdx,
+                                                 rdx,
+                                                 &operand_conversion_failure);
+        FloatingPointHelper::LoadInt32OperandFPU(masm,
+                                                 rcx,
+                                                 rbx,
+                                                 &operand_conversion_failure);
+      }
+      // 32-bit values in rbx, rcx.
 
-      Label skip_allocation, non_smi_result, operand_conversion_failure;
-
-      // Reserve space for converted numbers.
-      __ subq(rsp, Immediate(2 * kPointerSize));
-
-      if (use_sse3_) {
-        // Truncate the operands to 32-bit integers and check for
-        // exceptions in doing so.
-        CpuFeatures::Scope scope(CpuFeatures::SSE3);
-        __ fisttp_s(Operand(rsp, 0 * kPointerSize));
-        __ fisttp_s(Operand(rsp, 1 * kPointerSize));
-        __ fnstsw_ax();
-        __ testl(rax, Immediate(1));
-        __ j(not_zero, &operand_conversion_failure);
-      } else {
-        // Check if right operand is int32.
-        __ fist_s(Operand(rsp, 0 * kPointerSize));
-        __ fild_s(Operand(rsp, 0 * kPointerSize));
-        __ fucompp();
-        __ fnstsw_ax();
-        if (CpuFeatures::IsSupported(CpuFeatures::SAHF)) {
-          __ sahf();
-          __ j(not_zero, &operand_conversion_failure);
-          __ j(parity_even, &operand_conversion_failure);
-        } else {
-          __ and_(rax, Immediate(0x4400));
-          __ cmpl(rax, Immediate(0x4000));
-          __ j(not_zero, &operand_conversion_failure);
-        }
-        // Check if left operand is int32.
-        __ fist_s(Operand(rsp, 1 * kPointerSize));
-        __ fild_s(Operand(rsp, 1 * kPointerSize));
-        __ fucompp();
-        __ fnstsw_ax();
-        if (CpuFeatures::IsSupported(CpuFeatures::SAHF)) {
-          __ sahf();
-          __ j(not_zero, &operand_conversion_failure);
-          __ j(parity_even, &operand_conversion_failure);
-        } else {
-          __ and_(rax, Immediate(0x4400));
-          __ cmpl(rax, Immediate(0x4000));
-          __ j(not_zero, &operand_conversion_failure);
-        }
-      }
-
-      // Get int32 operands and perform bitop.
-      __ pop(rcx);
-      __ pop(rax);
       switch (op_) {
-        case Token::BIT_OR:  __ or_(rax, rcx); break;
-        case Token::BIT_AND: __ and_(rax, rcx); break;
-        case Token::BIT_XOR: __ xor_(rax, rcx); break;
-        case Token::SAR: __ sarl(rax); break;
-        case Token::SHL: __ shll(rax); break;
-        case Token::SHR: __ shrl(rax); break;
+        case Token::BIT_OR:  __ orl(rdx, rcx); break;
+        case Token::BIT_AND: __ andl(rdx, rcx); break;
+        case Token::BIT_XOR: __ xorl(rdx, rcx); break;
+        case Token::SAR: __ sarl(rdx); break;
+        case Token::SHL: __ shll(rdx); break;
+        case Token::SHR: __ shrl(rdx); break;
         default: UNREACHABLE();
       }
       if (op_ == Token::SHR) {
         // Check if result is non-negative and fits in a smi.
-        __ testl(rax, Immediate(0xc0000000));
+        __ testl(rdx, Immediate(0xc0000000));
         __ j(not_zero, &non_smi_result);
       } else {
         // Check if result fits in a smi.
-        __ cmpl(rax, Immediate(0xc0000000));
+        __ cmpl(rdx, Immediate(0xc0000000));
         __ j(negative, &non_smi_result);
       }
       // Tag smi result and return.
-      __ Integer32ToSmi(rax, rax);
+      __ Integer32ToSmi(rax, rdx);
       __ ret(2 * kPointerSize);
 
       // All ops except SHR return a signed int32 that we load in a HeapNumber.
       if (op_ != Token::SHR) {
         __ bind(&non_smi_result);
         // Allocate a heap number if needed.
-        __ movsxlq(rbx, rax);  // rbx: sign extended 32-bit result
+        __ movsxlq(rbx, rdx);  // rbx: sign extended 32-bit result
         switch (mode_) {
           case OVERWRITE_LEFT:
           case OVERWRITE_RIGHT:
             // If the operand was an object, we skip the
             // allocation of a heap number.
             __ movq(rax, Operand(rsp, mode_ == OVERWRITE_RIGHT ?
                                  1 * kPointerSize : 2 * kPointerSize));
             __ JumpIfNotSmi(rax, &skip_allocation);
             // Fall through!
           case NO_OVERWRITE:
             FloatingPointHelper::AllocateHeapNumber(masm, &call_runtime,
                                                     rcx, rax);
             __ bind(&skip_allocation);
             break;
           default: UNREACHABLE();
         }
         // Store the result in the HeapNumber and return.
         __ movq(Operand(rsp, 1 * kPointerSize), rbx);
         __ fild_s(Operand(rsp, 1 * kPointerSize));
         __ fstp_d(FieldOperand(rax, HeapNumber::kValueOffset));
         __ ret(2 * kPointerSize);
       }
 
-      // Clear the FPU exception flag and reset the stack before calling
-      // the runtime system.
+      // Fast-case operation failed, so call the runtime to perform the operation.
       __ bind(&operand_conversion_failure);
-      __ addq(rsp, Immediate(2 * kPointerSize));
-      if (use_sse3_) {
-        // If we've used the SSE3 instructions for truncating the
-        // floating point values to integers and it failed, we have a
-        // pending #IA exception. Clear it.
-        __ fnclex();
-      } else {
-        // The non-SSE3 variant does early bailout if the right
-        // operand isn't a 32-bit integer, so we may have a single
-        // value on the FPU stack we need to get rid of.
-        __ ffree(0);
-      }
 
       // SHR should return uint32 - go to runtime for non-smi/negative result.
       if (op_ == Token::SHR) {
         __ bind(&non_smi_result);
       }
       __ movq(rax, Operand(rsp, 1 * kPointerSize));
       __ movq(rdx, Operand(rsp, 2 * kPointerSize));
       break;
     }
     default: UNREACHABLE(); break;
@@ skipping 45 matching lines @@
 int CompareStub::MinorKey() {
   // Encode the two parameters in a unique 16 bit value.
   ASSERT(static_cast<unsigned>(cc_) < (1 << 15));
   return (static_cast<unsigned>(cc_) << 1) | (strict_ ? 1 : 0);
 }
 
 
 #undef __
 
 } }  // namespace v8::internal