X64: Load HeapNumberMap once in BinaryOpStub, and reuse it.

src/x64/codegen-x64.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 296 matching lines @@
   // Returns operands as floating point numbers on fp stack.
   static void LoadFloatOperands(MacroAssembler* masm,
                                 Register lhs,
                                 Register rhs);
 
   // Test if operands are smi or number objects (fp). Requirements:
   // operand_1 in rax, operand_2 in rdx; falls through on float or smi
   // operands, jumps to the non_float label otherwise.
   static void CheckNumberOperands(MacroAssembler* masm,
                                   Label* non_float);
+  // As CheckNumberOperands above, but expects the HeapNumber map in
+  // a register.
+  static void CheckNumberOperands(MacroAssembler* masm,
+                                  Label* non_float,
+                                  Register heap_number_map);
 
   // Takes the operands in rdx and rax and loads them as integers in rax
   // and rcx.
   static void LoadAsIntegers(MacroAssembler* masm,
-                             Label* operand_conversion_failure);
+                             Label* operand_conversion_failure,
+                             Register heap_number_map);
 };
 
 
 // -----------------------------------------------------------------------------
 // CodeGenerator implementation.
 
 CodeGenerator::CodeGenerator(MacroAssembler* masm)
     : deferred_(8),
       masm_(masm),
       info_(NULL),
@@ skipping 9649 matching lines @@
   __ fild_s(Operand(rsp, 0));
   __ pop(number);
 
   __ bind(&done);
 }
 
 
 void FloatingPointHelper::LoadFloatOperand(MacroAssembler* masm,
                                            Register src,
                                            XMMRegister dst) {
+  ASSERT(!src.is(kScratchRegister));
   Label load_smi, done;
 
   __ JumpIfSmi(src, &load_smi);
   __ movsd(dst, FieldOperand(src, HeapNumber::kValueOffset));
   __ jmp(&done);
 
   __ bind(&load_smi);
-  __ SmiToInteger32(src, src);
-  __ cvtlsi2sd(dst, src);
+  __ SmiToInteger32(kScratchRegister, src);
+  __ cvtlsi2sd(dst, kScratchRegister);
 
   __ bind(&done);
 }
 
 
 void FloatingPointHelper::LoadFloatOperand(MacroAssembler* masm,
                                            Register src,
                                            XMMRegister dst,
                                            Label* not_number) {
   Label load_smi, done;
   ASSERT(!src.is(kScratchRegister));
   __ JumpIfSmi(src, &load_smi);
   __ LoadRoot(kScratchRegister, Heap::kHeapNumberMapRootIndex);
   __ cmpq(FieldOperand(src, HeapObject::kMapOffset), kScratchRegister);
   __ j(not_equal, not_number);
   __ movsd(dst, FieldOperand(src, HeapNumber::kValueOffset));
   __ jmp(&done);
 
   __ bind(&load_smi);
   __ SmiToInteger32(kScratchRegister, src);
   __ cvtlsi2sd(dst, kScratchRegister);
 
   __ bind(&done);
 }
 
 
 void FloatingPointHelper::LoadFloatOperands(MacroAssembler* masm,
                                             XMMRegister dst1,
                                             XMMRegister dst2) {
-  __ movq(kScratchRegister, rdx);
-  LoadFloatOperand(masm, kScratchRegister, dst1);
-  __ movq(kScratchRegister, rax);
-  LoadFloatOperand(masm, kScratchRegister, dst2);
+  LoadFloatOperand(masm, rdx, dst1);
+  LoadFloatOperand(masm, rax, dst2);
 }
 
 
 void FloatingPointHelper::LoadFloatOperandsFromSmis(MacroAssembler* masm,
                                                     XMMRegister dst1,
                                                     XMMRegister dst2) {
   __ SmiToInteger32(kScratchRegister, rdx);
   __ cvtlsi2sd(dst1, kScratchRegister);
   __ SmiToInteger32(kScratchRegister, rax);
   __ cvtlsi2sd(dst2, kScratchRegister);
 }
 
 
 // Input: rdx, rax are the left and right objects of a bit op.
 // Output: rax, rcx are left and right integers for a bit op.
 void FloatingPointHelper::LoadAsIntegers(MacroAssembler* masm,
-                                         Label* conversion_failure) {
+                                         Label* conversion_failure,
+                                         Register heap_number_map) {
   // Check float operands.
   Label arg1_is_object, check_undefined_arg1;
   Label arg2_is_object, check_undefined_arg2;
   Label load_arg2, done;
 
   __ JumpIfNotSmi(rdx, &arg1_is_object);
   __ SmiToInteger32(rdx, rdx);
   __ jmp(&load_arg2);
 
   // If the argument is undefined it converts to zero (ECMA-262, section 9.5).
   __ bind(&check_undefined_arg1);
   __ CompareRoot(rdx, Heap::kUndefinedValueRootIndex);
   __ j(not_equal, conversion_failure);
   __ movl(rdx, Immediate(0));
   __ jmp(&load_arg2);
 
   __ bind(&arg1_is_object);
-  __ movq(rbx, FieldOperand(rdx, HeapObject::kMapOffset));
-  __ CompareRoot(rbx, Heap::kHeapNumberMapRootIndex);
+  __ cmpq(FieldOperand(rdx, HeapObject::kMapOffset), heap_number_map);
   __ j(not_equal, &check_undefined_arg1);
   // Get the untagged integer version of the edx heap number in rcx.
   IntegerConvert(masm, rdx, rdx);
 
   // Here rdx has the untagged integer, rax has a Smi or a heap number.
   __ bind(&load_arg2);
   // Test if arg2 is a Smi.
   __ JumpIfNotSmi(rax, &arg2_is_object);
   __ SmiToInteger32(rax, rax);
   __ movl(rcx, rax);
   __ jmp(&done);
 
   // If the argument is undefined it converts to zero (ECMA-262, section 9.5).
   __ bind(&check_undefined_arg2);
   __ CompareRoot(rax, Heap::kUndefinedValueRootIndex);
   __ j(not_equal, conversion_failure);
   __ movl(rcx, Immediate(0));
   __ jmp(&done);
 
   __ bind(&arg2_is_object);
-  __ movq(rbx, FieldOperand(rax, HeapObject::kMapOffset));
-  __ CompareRoot(rbx, Heap::kHeapNumberMapRootIndex);
+  __ cmpq(FieldOperand(rax, HeapObject::kMapOffset), heap_number_map);
   __ j(not_equal, &check_undefined_arg2);
   // Get the untagged integer version of the eax heap number in ecx.
   IntegerConvert(masm, rcx, rax);
   __ bind(&done);
   __ movl(rax, rdx);
 }
 
 
 void FloatingPointHelper::LoadFloatOperands(MacroAssembler* masm,
                                             Register lhs,
@@ skipping 36 matching lines @@
   __ bind(&test_other);
   __ JumpIfSmi(rax, &done);  // argument in rax is OK
   __ Cmp(FieldOperand(rax, HeapObject::kMapOffset), Factory::heap_number_map());
   __ j(not_equal, non_float);  // The argument in rax is not a number.
 
   // Fall-through: Both operands are numbers.
   __ bind(&done);
 }
 
 
+void FloatingPointHelper::CheckNumberOperands(MacroAssembler* masm,
+                                              Label* non_float,
+                                              Register heap_number_map) {
+  Label test_other, done;
+  // Test if both operands are numbers (heap_numbers or smis).
+  // If not, jump to label non_float.
+  __ JumpIfSmi(rdx, &test_other);  // argument in rdx is OK
+  __ cmpq(FieldOperand(rdx, HeapObject::kMapOffset), heap_number_map);
+  __ j(not_equal, non_float);  // The argument in rdx is not a number.
+
+  __ bind(&test_other);
+  __ JumpIfSmi(rax, &done);  // argument in rax is OK
+  __ cmpq(FieldOperand(rax, HeapObject::kMapOffset), heap_number_map);
+  __ j(not_equal, non_float);  // The argument in rax is not a number.
+
+  // Fall-through: Both operands are numbers.
+  __ bind(&done);
+}
+
+
 const char* GenericBinaryOpStub::GetName() {
   if (name_ != NULL) return name_;
   const int len = 100;
   name_ = Bootstrapper::AllocateAutoDeletedArray(len);
   if (name_ == NULL) return "OOM";
   const char* op_name = Token::Name(op_);
   const char* overwrite_name;
   switch (mode_) {
     case NO_OVERWRITE: overwrite_name = "Alloc"; break;
     case OVERWRITE_RIGHT: overwrite_name = "OverwriteRight"; break;
@@ skipping 339 matching lines @@
 
   if (ShouldGenerateSmiCode()) {
     GenerateSmiCode(masm, &call_runtime);
   } else if (op_ != Token::MOD) {
     if (!HasArgsInRegisters()) {
       GenerateLoadArguments(masm);
     }
   }
   // Floating point case.
   if (ShouldGenerateFPCode()) {
+    // Load the HeapNumber map here and use it throughout the FP code.
+    Register heap_number_map = r9;
     switch (op_) {
       case Token::ADD:
       case Token::SUB:
       case Token::MUL:
       case Token::DIV: {
         if (runtime_operands_type_ == BinaryOpIC::DEFAULT &&
             HasSmiCodeInStub()) {
           // Execution reaches this point when the first non-smi argument occurs
           // (and only if smi code is generated). This is the right moment to
           // patch to HEAP_NUMBERS state. The transition is attempted only for
           // the four basic operations. The stub stays in the DEFAULT state
           // forever for all other operations (also if smi code is skipped).
           GenerateTypeTransition(masm);
         }
+        __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
 
         Label not_floats;
         // rax: y
         // rdx: x
-      if (static_operands_type_.IsNumber() && FLAG_debug_code) {
-        // Assert at runtime that inputs are only numbers.
-        __ AbortIfNotNumber(rdx);
-        __ AbortIfNotNumber(rax);
-      } else {
-        FloatingPointHelper::CheckNumberOperands(masm, &call_runtime);
-      }
+        if (static_operands_type_.IsNumber() && FLAG_debug_code) {
+          // Assert at runtime that inputs are only numbers.
+          __ AbortIfNotNumber(rdx);
+          __ AbortIfNotNumber(rax);
+        } else {
+          if (FLAG_debug_code) {
+            __ AbortIfNotRootValue(heap_number_map,
+                                   Heap::kHeapNumberMapRootIndex,
+                                   "HeapNumberMap register clobbered.");
+          }
+          FloatingPointHelper::CheckNumberOperands(masm,
+                                                   &call_runtime,
+                                                   heap_number_map);
+        }
         // Fast-case: Both operands are numbers.
         // xmm4 and xmm5 are volatile XMM registers.
         FloatingPointHelper::LoadFloatOperands(masm, xmm4, xmm5);
 
         switch (op_) {
           case Token::ADD: __ addsd(xmm4, xmm5); break;
           case Token::SUB: __ subsd(xmm4, xmm5); break;
           case Token::MUL: __ mulsd(xmm4, xmm5); break;
           case Token::DIV: __ divsd(xmm4, xmm5); break;
           default: UNREACHABLE();
         }
         // Allocate a heap number, if needed.
         Label skip_allocation;
         OverwriteMode mode = mode_;
         if (HasArgsReversed()) {
           if (mode == OVERWRITE_RIGHT) {
             mode = OVERWRITE_LEFT;
           } else if (mode == OVERWRITE_LEFT) {
             mode = OVERWRITE_RIGHT;
           }
         }
         switch (mode) {
+          // TODO(lrn): Allocate this when we first see that the
+          // left register is a smi (and load it into xmm4).
           case OVERWRITE_LEFT:
             __ JumpIfNotSmi(rdx, &skip_allocation);
-            __ AllocateHeapNumber(rbx, rcx, &call_runtime);
+            // Allocate heap number in new space.
+            __ AllocateInNewSpace(HeapNumber::kSize,
+                                  rbx,
+                                  rcx,
+                                  no_reg,
+                                  &call_runtime,
+                                  TAG_OBJECT);
+            if (FLAG_debug_code) {
+              __ AbortIfNotRootValue(heap_number_map,
+                                     Heap::kHeapNumberMapRootIndex,
+                                     "HeapNumberMap register clobbered.");
+            }
+            __ movq(FieldOperand(rbx, HeapObject::kMapOffset),
+                    heap_number_map);
             __ movq(rdx, rbx);
             __ bind(&skip_allocation);
             __ movq(rax, rdx);
             break;
           case OVERWRITE_RIGHT:
             // If the argument in rax is already an object, we skip the
             // allocation of a heap number.
+            // TODO(lrn): Allocate the heap number when we first see that the
+            // right register is a smi (and load it into xmm5).
             __ JumpIfNotSmi(rax, &skip_allocation);
             // Fall through!
           case NO_OVERWRITE:
             // Allocate a heap number for the result. Keep rax and rdx intact
             // for the possible runtime call.
-            __ AllocateHeapNumber(rbx, rcx, &call_runtime);
+            __ AllocateInNewSpace(HeapNumber::kSize,
+                                  rbx,
+                                  rcx,
+                                  no_reg,
+                                  &call_runtime,
+                                  TAG_OBJECT);
+            if (FLAG_debug_code) {
+              __ AbortIfNotRootValue(heap_number_map,
+                                     Heap::kHeapNumberMapRootIndex,
+                                     "HeapNumberMap register clobbered.");
+            }
+            __ movq(FieldOperand(rbx, HeapObject::kMapOffset),
+                    heap_number_map);
             __ movq(rax, rbx);
             __ bind(&skip_allocation);
             break;
           default: UNREACHABLE();
         }
         __ movsd(FieldOperand(rax, HeapNumber::kValueOffset), xmm4);
         GenerateReturn(masm);
         __ bind(&not_floats);
         if (runtime_operands_type_ == BinaryOpIC::DEFAULT &&
             !HasSmiCodeInStub()) {
@@ skipping 10 matching lines @@
       case Token::MOD: {
         // For MOD we go directly to runtime in the non-smi case.
         break;
       }
       case Token::BIT_OR:
       case Token::BIT_AND:
       case Token::BIT_XOR:
       case Token::SAR:
       case Token::SHL:
       case Token::SHR: {
-        Label skip_allocation, non_smi_result;
-        FloatingPointHelper::LoadAsIntegers(masm, &call_runtime);
+        Label skip_allocation, non_smi_shr_result;
+        __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
+        FloatingPointHelper::LoadAsIntegers(masm,
+                                            &call_runtime,
+                                            heap_number_map);
         switch (op_) {
           case Token::BIT_OR:  __ orl(rax, rcx); break;
           case Token::BIT_AND: __ andl(rax, rcx); break;
           case Token::BIT_XOR: __ xorl(rax, rcx); break;
           case Token::SAR: __ sarl_cl(rax); break;
           case Token::SHL: __ shll_cl(rax); break;
           case Token::SHR: __ shrl_cl(rax); break;
           default: UNREACHABLE();
         }
         if (op_ == Token::SHR) {
           // Check if result is negative. This can only happen for a shift
-          // by zero, which also doesn't update the sign flag.
+          // by zero.
           __ testl(rax, rax);
-          __ j(negative, &non_smi_result);
+          __ j(negative, &non_smi_shr_result);
         }
-        __ JumpIfNotValidSmiValue(rax, &non_smi_result);
-        // Tag smi result, if possible, and return.
+        STATIC_ASSERT(kSmiValueSize == 32);
+        // Tag smi result and return.
         __ Integer32ToSmi(rax, rax);
         GenerateReturn(masm);
 
-        // All ops except SHR return a signed int32 that we load in
-        // a HeapNumber.
-        if (op_ != Token::SHR && non_smi_result.is_linked()) {
-          __ bind(&non_smi_result);
+        // All bit-ops except SHR return a signed int32 that can be
+        // returned immediately as a smi.
+        if (op_ == Token::SHR) {
+          ASSERT(non_smi_shr_result.is_linked());
+          __ bind(&non_smi_shr_result);
           // Allocate a heap number if needed.
-          __ movsxlq(rbx, rax);  // rbx: sign extended 32-bit result
+          __ movl(rbx, rax);  // rbx holds result value (uint32 value as int64).
           switch (mode_) {
             case OVERWRITE_LEFT:
             case OVERWRITE_RIGHT:
               // If the operand was an object, we skip the
               // allocation of a heap number.
               __ movq(rax, Operand(rsp, mode_ == OVERWRITE_RIGHT ?
                                    1 * kPointerSize : 2 * kPointerSize));
               __ JumpIfNotSmi(rax, &skip_allocation);
               // Fall through!
             case NO_OVERWRITE:
-              __ AllocateHeapNumber(rax, rcx, &call_runtime);
+              // Allocate heap number in new space.
+              __ AllocateInNewSpace(HeapNumber::kSize,
+                                    rax,
+                                    rcx,
+                                    no_reg,
+                                    &call_runtime,
+                                    TAG_OBJECT);
+              // Set the map.
+              if (FLAG_debug_code) {
+                __ AbortIfNotRootValue(heap_number_map,
+                                       Heap::kHeapNumberMapRootIndex,
+                                       "HeapNumberMap register clobbered.");
+              }
+              __ movq(FieldOperand(rax, HeapObject::kMapOffset),
+                      heap_number_map);
               __ bind(&skip_allocation);
               break;
             default: UNREACHABLE();
           }
           // Store the result in the HeapNumber and return.
-          __ movq(Operand(rsp, 1 * kPointerSize), rbx);
-          __ fild_s(Operand(rsp, 1 * kPointerSize));
-          __ fstp_d(FieldOperand(rax, HeapNumber::kValueOffset));
+          __ cvtqsi2sd(xmm0, rbx);
+          __ movsd(FieldOperand(rax, HeapNumber::kValueOffset), xmm0);
           GenerateReturn(masm);
         }
 
-        // SHR should return uint32 - go to runtime for non-smi/negative result.
-        if (op_ == Token::SHR) {
-          __ bind(&non_smi_result);
-        }
         break;
       }
       default: UNREACHABLE(); break;
     }
   }
 
   // If all else fails, use the runtime system to get the correct
   // result. If arguments was passed in registers now place them on the
   // stack in the correct order below the return address.
   __ bind(&call_runtime);
@@ skipping 12 matching lines @@
         rhs = rdx;
       } else {
         lhs = rdx;
         rhs = rax;
       }
 
       // Test for string arguments before calling runtime.
       Label not_strings, both_strings, not_string1, string1, string1_smi2;
 
       // If this stub has already generated FP-specific code then the arguments
-      // are already in rdx, rax
+      // are already in rdx, rax.
       if (!ShouldGenerateFPCode() && !HasArgsInRegisters()) {
         GenerateLoadArguments(masm);
       }
 
       Condition is_smi;
       is_smi = masm->CheckSmi(lhs);
       __ j(is_smi, &not_string1);
       __ CmpObjectType(lhs, FIRST_NONSTRING_TYPE, r8);
       __ j(above_equal, &not_string1);
 
@@ skipping 1260 matching lines @@
 }
 
 #endif
 
 
 #undef __
 
 } }  // namespace v8::internal
 
 #endif  // V8_TARGET_ARCH_X64