Lattice-based representation inference, powered by left/right specific type feedback for BinaryOps …

src/x64/code-stubs-x64.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 979 matching lines @@
     case UNARY_NO_OVERWRITE: overwrite_name = "Alloc"; break;
     case UNARY_OVERWRITE: overwrite_name = "Overwrite"; break;
   }
   stream->Add("UnaryOpStub_%s_%s_%s",
               op_name,
               overwrite_name,
               UnaryOpIC::GetName(operand_type_));
 }
 
 
+void BinaryOpStub::Initialize() {}
+
+
 void BinaryOpStub::GenerateTypeTransition(MacroAssembler* masm) {
   __ pop(rcx);  // Save return address.
   __ push(rdx);
   __ push(rax);
   // Left and right arguments are now on top.
-  // Push this stub's key. Although the operation and the type info are
-  // encoded into the key, the encoding is opaque, so push them too.
   __ Push(Smi::FromInt(MinorKey()));
-  __ Push(Smi::FromInt(op_));
-  __ Push(Smi::FromInt(operands_type_));
 
   __ push(rcx);  // Push return address.
 
   // Patch the caller to an appropriate specialized stub and return the
   // operation result to the caller of the stub.
   __ TailCallExternalReference(
       ExternalReference(IC_Utility(IC::kBinaryOp_Patch),
                         masm->isolate()),
-      5,
+      3,
       1);
 }
 
 
-void BinaryOpStub::Generate(MacroAssembler* masm) {
-  // Explicitly allow generation of nested stubs. It is safe here because
-  // generation code does not use any raw pointers.
-  AllowStubCallsScope allow_stub_calls(masm, true);
-
-  switch (operands_type_) {
-    case BinaryOpIC::UNINITIALIZED:
-      GenerateTypeTransition(masm);
-      break;
-    case BinaryOpIC::SMI:
-      GenerateSmiStub(masm);
-      break;
-    case BinaryOpIC::INT32:
-      UNREACHABLE();
-      // The int32 case is identical to the Smi case.  We avoid creating this
-      // ic state on x64.
-      break;
-    case BinaryOpIC::HEAP_NUMBER:
-      GenerateHeapNumberStub(masm);
-      break;
-    case BinaryOpIC::ODDBALL:
-      GenerateOddballStub(masm);
-      break;
-    case BinaryOpIC::BOTH_STRING:
-      GenerateBothStringStub(masm);
-      break;
-    case BinaryOpIC::STRING:
-      GenerateStringStub(masm);
-      break;
-    case BinaryOpIC::GENERIC:
-      GenerateGeneric(masm);
-      break;
-    default:
-      UNREACHABLE();
-  }
-}
-
-
-void BinaryOpStub::PrintName(StringStream* stream) {
-  const char* op_name = Token::Name(op_);
-  const char* overwrite_name;
-  switch (mode_) {
-    case NO_OVERWRITE: overwrite_name = "Alloc"; break;
-    case OVERWRITE_RIGHT: overwrite_name = "OverwriteRight"; break;
-    case OVERWRITE_LEFT: overwrite_name = "OverwriteLeft"; break;
-    default: overwrite_name = "UnknownOverwrite"; break;
-  }
-  stream->Add("BinaryOpStub_%s_%s_%s",
-              op_name,
-              overwrite_name,
-              BinaryOpIC::GetName(operands_type_));
-}
-
-
-void BinaryOpStub::GenerateSmiCode(
+static void BinaryOpStub_GenerateSmiCode(
     MacroAssembler* masm,
     Label* slow,
-    SmiCodeGenerateHeapNumberResults allow_heapnumber_results) {
+    BinaryOpStub::SmiCodeGenerateHeapNumberResults allow_heapnumber_results,
+    Token::Value op) {
 
   // Arguments to BinaryOpStub are in rdx and rax.
   const Register left = rdx;
   const Register right = rax;
 
   // We only generate heapnumber answers for overflowing calculations
   // for the four basic arithmetic operations and logical right shift by 0.
   bool generate_inline_heapnumber_results =
-      (allow_heapnumber_results == ALLOW_HEAPNUMBER_RESULTS) &&
-      (op_ == Token::ADD || op_ == Token::SUB ||
-       op_ == Token::MUL || op_ == Token::DIV || op_ == Token::SHR);
+      (allow_heapnumber_results == BinaryOpStub::ALLOW_HEAPNUMBER_RESULTS) &&
+      (op == Token::ADD || op == Token::SUB ||
+       op == Token::MUL || op == Token::DIV || op == Token::SHR);
 
   // Smi check of both operands.  If op is BIT_OR, the check is delayed
   // until after the OR operation.
   Label not_smis;
   Label use_fp_on_smis;
   Label fail;
 
-  if (op_ != Token::BIT_OR) {
+  if (op != Token::BIT_OR) {
     Comment smi_check_comment(masm, "-- Smi check arguments");
     __ JumpIfNotBothSmi(left, right, &not_smis);
   }
 
   Label smi_values;
   __ bind(&smi_values);
   // Perform the operation.
   Comment perform_smi(masm, "-- Perform smi operation");
-  switch (op_) {
+  switch (op) {
     case Token::ADD:
       ASSERT(right.is(rax));
       __ SmiAdd(right, right, left, &use_fp_on_smis);  // ADD is commutative.
       break;
 
     case Token::SUB:
       __ SmiSub(left, left, right, &use_fp_on_smis);
       __ movq(rax, left);
       break;
 
@@ skipping 51 matching lines @@
   }
 
   // 5. Emit return of result in rax.  Some operations have registers pushed.
   __ ret(0);
 
   if (use_fp_on_smis.is_linked()) {
     // 6. For some operations emit inline code to perform floating point
     //    operations on known smis (e.g., if the result of the operation
     //    overflowed the smi range).
     __ bind(&use_fp_on_smis);
-    if (op_ == Token::DIV || op_ == Token::MOD) {
+    if (op == Token::DIV || op == Token::MOD) {
       // Restore left and right to rdx and rax.
       __ movq(rdx, rcx);
       __ movq(rax, rbx);
     }
 
     if (generate_inline_heapnumber_results) {
       __ AllocateHeapNumber(rcx, rbx, slow);
       Comment perform_float(masm, "-- Perform float operation on smis");
-      if (op_ == Token::SHR) {
+      if (op == Token::SHR) {
         __ SmiToInteger32(left, left);
         __ cvtqsi2sd(xmm0, left);
       } else {
         FloatingPointHelper::LoadSSE2SmiOperands(masm);
-        switch (op_) {
+        switch (op) {
         case Token::ADD: __ addsd(xmm0, xmm1); break;
         case Token::SUB: __ subsd(xmm0, xmm1); break;
         case Token::MUL: __ mulsd(xmm0, xmm1); break;
         case Token::DIV: __ divsd(xmm0, xmm1); break;
         default: UNREACHABLE();
         }
       }
       __ movsd(FieldOperand(rcx, HeapNumber::kValueOffset), xmm0);
       __ movq(rax, rcx);
       __ ret(0);
     } else {
       __ jmp(&fail);
     }
   }
 
   // 7. Non-smi operands reach the end of the code generated by
   //    GenerateSmiCode, and fall through to subsequent code,
   //    with the operands in rdx and rax.
   //    But first we check if non-smi values are HeapNumbers holding
   //    values that could be smi.
   __ bind(&not_smis);
   Comment done_comment(masm, "-- Enter non-smi code");
   FloatingPointHelper::NumbersToSmis(masm, left, right, rbx, rdi, rcx,
                                      &smi_values, &fail);
   __ jmp(&smi_values);
   __ bind(&fail);
 }
 
 
-void BinaryOpStub::GenerateFloatingPointCode(MacroAssembler* masm,
-                                             Label* allocation_failure,
-                                             Label* non_numeric_failure) {
-  switch (op_) {
+static void BinaryOpStub_GenerateHeapResultAllocation(MacroAssembler* masm,
+                                                      Label* alloc_failure,
+                                                      OverwriteMode mode);
+
+
+static void BinaryOpStub_GenerateFloatingPointCode(MacroAssembler* masm,
+                                                   Label* allocation_failure,
+                                                   Label* non_numeric_failure,
+                                                   Token::Value op,
+                                                   OverwriteMode mode) {
+  switch (op) {
     case Token::ADD:
     case Token::SUB:
     case Token::MUL:
     case Token::DIV: {
       FloatingPointHelper::LoadSSE2UnknownOperands(masm, non_numeric_failure);
 
-      switch (op_) {
+      switch (op) {
         case Token::ADD: __ addsd(xmm0, xmm1); break;
         case Token::SUB: __ subsd(xmm0, xmm1); break;
         case Token::MUL: __ mulsd(xmm0, xmm1); break;
         case Token::DIV: __ divsd(xmm0, xmm1); break;
         default: UNREACHABLE();
       }
-      GenerateHeapResultAllocation(masm, allocation_failure);
+      BinaryOpStub_GenerateHeapResultAllocation(
+          masm, allocation_failure, mode);
       __ movsd(FieldOperand(rax, HeapNumber::kValueOffset), xmm0);
       __ ret(0);
       break;
     }
     case Token::MOD: {
       // For MOD we jump to the allocation_failure label, to call runtime.
       __ jmp(allocation_failure);
       break;
     }
     case Token::BIT_OR:
     case Token::BIT_AND:
     case Token::BIT_XOR:
     case Token::SAR:
     case Token::SHL:
     case Token::SHR: {
       Label non_smi_shr_result;
       Register heap_number_map = r9;
       __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
       FloatingPointHelper::LoadAsIntegers(masm, non_numeric_failure,
                                           heap_number_map);
-      switch (op_) {
+      switch (op) {
         case Token::BIT_OR:  __ orl(rax, rcx); break;
         case Token::BIT_AND: __ andl(rax, rcx); break;
         case Token::BIT_XOR: __ xorl(rax, rcx); break;
         case Token::SAR: __ sarl_cl(rax); break;
         case Token::SHL: __ shll_cl(rax); break;
         case Token::SHR: {
           __ shrl_cl(rax);
           // Check if result is negative. This can only happen for a shift
           // by zero.
           __ testl(rax, rax);
           __ j(negative, &non_smi_shr_result);
           break;
         }
         default: UNREACHABLE();
       }
       STATIC_ASSERT(kSmiValueSize == 32);
       // Tag smi result and return.
       __ Integer32ToSmi(rax, rax);
       __ Ret();
 
       // Logical shift right can produce an unsigned int32 that is not
       // an int32, and so is not in the smi range.  Allocate a heap number
       // in that case.
-      if (op_ == Token::SHR) {
+      if (op == Token::SHR) {
         __ bind(&non_smi_shr_result);
         Label allocation_failed;
         __ movl(rbx, rax);  // rbx holds result value (uint32 value as int64).
         // Allocate heap number in new space.
         // Not using AllocateHeapNumber macro in order to reuse
         // already loaded heap_number_map.
         __ AllocateInNewSpace(HeapNumber::kSize,
                               rax,
                               rdx,
                               no_reg,
@@ skipping 16 matching lines @@
         __ Integer32ToSmi(rdx, rbx);
         __ jmp(allocation_failure);
       }
       break;
     }
     default: UNREACHABLE(); break;
   }
   // No fall-through from this generated code.
   if (FLAG_debug_code) {
     __ Abort("Unexpected fall-through in "
-             "BinaryStub::GenerateFloatingPointCode.");
+             "BinaryStub_GenerateFloatingPointCode.");
   }
 }
 
 
-void BinaryOpStub::GenerateStringAddCode(MacroAssembler* masm) {
+void BinaryOpStub::GenerateAddStrings(MacroAssembler* masm) {
   ASSERT(op_ == Token::ADD);
   Label left_not_string, call_runtime;
 
   // Registers containing left and right operands respectively.
   Register left = rdx;
   Register right = rax;
 
   // Test if left operand is a string.
   __ JumpIfSmi(left, &left_not_string, Label::kNear);
   __ CmpObjectType(left, FIRST_NONSTRING_TYPE, rcx);
@@ skipping 10 matching lines @@
 
   StringAddStub string_add_right_stub(NO_STRING_CHECK_RIGHT_IN_STUB);
   GenerateRegisterArgsPush(masm);
   __ TailCallStub(&string_add_right_stub);
 
   // Neither argument is a string.
   __ bind(&call_runtime);
 }
 
 
-void BinaryOpStub::GenerateCallRuntimeCode(MacroAssembler* masm) {
-  GenerateRegisterArgsPush(masm);
-  switch (op_) {
-    case Token::ADD:
-      __ InvokeBuiltin(Builtins::ADD, JUMP_FUNCTION);
-      break;
-    case Token::SUB:
-      __ InvokeBuiltin(Builtins::SUB, JUMP_FUNCTION);
-      break;
-    case Token::MUL:
-      __ InvokeBuiltin(Builtins::MUL, JUMP_FUNCTION);
-      break;
-    case Token::DIV:
-      __ InvokeBuiltin(Builtins::DIV, JUMP_FUNCTION);
-      break;
-    case Token::MOD:
-      __ InvokeBuiltin(Builtins::MOD, JUMP_FUNCTION);
-      break;
-    case Token::BIT_OR:
-      __ InvokeBuiltin(Builtins::BIT_OR, JUMP_FUNCTION);
-      break;
-    case Token::BIT_AND:
-      __ InvokeBuiltin(Builtins::BIT_AND, JUMP_FUNCTION);
-      break;
-    case Token::BIT_XOR:
-      __ InvokeBuiltin(Builtins::BIT_XOR, JUMP_FUNCTION);
-      break;
-    case Token::SAR:
-      __ InvokeBuiltin(Builtins::SAR, JUMP_FUNCTION);
-      break;
-    case Token::SHL:
-      __ InvokeBuiltin(Builtins::SHL, JUMP_FUNCTION);
-      break;
-    case Token::SHR:
-      __ InvokeBuiltin(Builtins::SHR, JUMP_FUNCTION);
-      break;
-    default:
-      UNREACHABLE();
-  }
-}
-
-
 void BinaryOpStub::GenerateSmiStub(MacroAssembler* masm) {
   Label call_runtime;
   if (result_type_ == BinaryOpIC::UNINITIALIZED ||
       result_type_ == BinaryOpIC::SMI) {
     // Only allow smi results.
-    GenerateSmiCode(masm, NULL, NO_HEAPNUMBER_RESULTS);
+    BinaryOpStub_GenerateSmiCode(masm, NULL, NO_HEAPNUMBER_RESULTS, op_);
   } else {
     // Allow heap number result and don't make a transition if a heap number
     // cannot be allocated.
-    GenerateSmiCode(masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS);
+    BinaryOpStub_GenerateSmiCode(
+        masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS, op_);
   }
 
   // Code falls through if the result is not returned as either a smi or heap
   // number.
   GenerateTypeTransition(masm);
 
   if (call_runtime.is_linked()) {
     __ bind(&call_runtime);
-    GenerateCallRuntimeCode(masm);
+    GenerateRegisterArgsPush(masm);
+    GenerateCallRuntime(masm);
   }
 }
 
 
-void BinaryOpStub::GenerateStringStub(MacroAssembler* masm) {
-  ASSERT(operands_type_ == BinaryOpIC::STRING);
-  ASSERT(op_ == Token::ADD);
-  GenerateStringAddCode(masm);
-  // Try to add arguments as strings, otherwise, transition to the generic
-  // BinaryOpIC type.
-  GenerateTypeTransition(masm);
+void BinaryOpStub::GenerateInt32Stub(MacroAssembler* masm) {
+  // The int32 case is identical to the Smi case.  We avoid creating this
+  // ic state on x64.
+  UNREACHABLE();
 }
 
 
 void BinaryOpStub::GenerateBothStringStub(MacroAssembler* masm) {
   Label call_runtime;
-  ASSERT(operands_type_ == BinaryOpIC::BOTH_STRING);
+  ASSERT(left_type_ == BinaryOpIC::STRING && right_type_ == BinaryOpIC::STRING);
   ASSERT(op_ == Token::ADD);
   // If both arguments are strings, call the string add stub.
   // Otherwise, do a transition.
 
   // Registers containing left and right operands respectively.
   Register left = rdx;
   Register right = rax;
 
   // Test if left operand is a string.
   __ JumpIfSmi(left, &call_runtime);
@@ skipping 13 matching lines @@
   GenerateTypeTransition(masm);
 }
 
 
 void BinaryOpStub::GenerateOddballStub(MacroAssembler* masm) {
   Label call_runtime;
 
   if (op_ == Token::ADD) {
     // Handle string addition here, because it is the only operation
     // that does not do a ToNumber conversion on the operands.
-    GenerateStringAddCode(masm);
+    GenerateAddStrings(masm);
   }
 
   // Convert oddball arguments to numbers.
   Label check, done;
   __ CompareRoot(rdx, Heap::kUndefinedValueRootIndex);
   __ j(not_equal, &check, Label::kNear);
   if (Token::IsBitOp(op_)) {
     __ xor_(rdx, rdx);
   } else {
     __ LoadRoot(rdx, Heap::kNanValueRootIndex);
   }
   __ jmp(&done, Label::kNear);
   __ bind(&check);
   __ CompareRoot(rax, Heap::kUndefinedValueRootIndex);
   __ j(not_equal, &done, Label::kNear);
   if (Token::IsBitOp(op_)) {
     __ xor_(rax, rax);
   } else {
     __ LoadRoot(rax, Heap::kNanValueRootIndex);
   }
   __ bind(&done);
 
   GenerateHeapNumberStub(masm);
 }
 
 
+static void BinaryOpStub_CheckSmiInput(MacroAssembler* masm,
+                                       Register input,
+                                       Label* fail) {
+  Label ok;
+  __ JumpIfSmi(input, &ok, Label::kNear);
+  Register heap_number_map = r8;
+  Register scratch1 = r9;
+  Register scratch2 = r10;
+  // HeapNumbers containing 32bit integer values are also allowed.
+  __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
+  __ cmpq(FieldOperand(input, HeapObject::kMapOffset), heap_number_map);
+  __ j(not_equal, fail);
+  __ movsd(xmm0, FieldOperand(input, HeapNumber::kValueOffset));
+  // Convert, convert back, and compare the two doubles' bits.
+  __ cvttsd2siq(scratch2, xmm0);
+  __ cvtlsi2sd(xmm1, scratch2);
+  __ movq(scratch1, xmm0);
+  __ movq(scratch2, xmm1);
+  __ cmpq(scratch1, scratch2);
+  __ j(not_equal, fail);
+  __ bind(&ok);
+}
+
+
 void BinaryOpStub::GenerateHeapNumberStub(MacroAssembler* masm) {
   Label gc_required, not_number;
-  GenerateFloatingPointCode(masm, &gc_required, &not_number);
+
+  // It could be that only SMIs have been seen at either the left
+  // or the right operand. For precise type feedback, patch the IC
+  // again if this changes.
+  if (left_type_ == BinaryOpIC::SMI) {
+    BinaryOpStub_CheckSmiInput(masm, rdx, &not_number);
+  }
+  if (right_type_ == BinaryOpIC::SMI) {
+    BinaryOpStub_CheckSmiInput(masm, rax, &not_number);
+  }
+
+  BinaryOpStub_GenerateFloatingPointCode(
+      masm, &gc_required, &not_number, op_, mode_);
 
   __ bind(&not_number);
   GenerateTypeTransition(masm);
 
   __ bind(&gc_required);
-  GenerateCallRuntimeCode(masm);
+  GenerateRegisterArgsPush(masm);
+  GenerateCallRuntime(masm);
 }
 
 
 void BinaryOpStub::GenerateGeneric(MacroAssembler* masm) {
   Label call_runtime, call_string_add_or_runtime;
 
-  GenerateSmiCode(masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS);
+  BinaryOpStub_GenerateSmiCode(
+      masm, &call_runtime, ALLOW_HEAPNUMBER_RESULTS, op_);
 
-  GenerateFloatingPointCode(masm, &call_runtime, &call_string_add_or_runtime);
+  BinaryOpStub_GenerateFloatingPointCode(
+      masm, &call_runtime, &call_string_add_or_runtime, op_, mode_);
 
   __ bind(&call_string_add_or_runtime);
   if (op_ == Token::ADD) {
-    GenerateStringAddCode(masm);
+    GenerateAddStrings(masm);
   }
 
   __ bind(&call_runtime);
-  GenerateCallRuntimeCode(masm);
+  GenerateRegisterArgsPush(masm);
+  GenerateCallRuntime(masm);
 }
 
 
-void BinaryOpStub::GenerateHeapResultAllocation(MacroAssembler* masm,
-                                                Label* alloc_failure) {
+static void BinaryOpStub_GenerateHeapResultAllocation(MacroAssembler* masm,
+                                                      Label* alloc_failure,
+                                                      OverwriteMode mode) {
   Label skip_allocation;
-  OverwriteMode mode = mode_;
   switch (mode) {
     case OVERWRITE_LEFT: {
       // If the argument in rdx is already an object, we skip the
       // allocation of a heap number.
       __ JumpIfNotSmi(rdx, &skip_allocation);
       // Allocate a heap number for the result. Keep eax and edx intact
       // for the possible runtime call.
       __ AllocateHeapNumber(rbx, rcx, alloc_failure);
       // Now rdx can be overwritten losing one of the arguments as we are
       // now done and will not need it any more.
@@ skipping 478 matching lines @@
 void FloatingPointHelper::NumbersToSmis(MacroAssembler* masm,
                                         Register first,
                                         Register second,
                                         Register scratch1,
                                         Register scratch2,
                                         Register scratch3,
                                         Label* on_success,
                                         Label* on_not_smis)   {
   Register heap_number_map = scratch3;
   Register smi_result = scratch1;
-  Label done;
+  Label done, maybe_undefined_first, maybe_undefined_second, first_done;
 
   __ LoadRoot(heap_number_map, Heap::kHeapNumberMapRootIndex);
 
   Label first_smi;
   __ JumpIfSmi(first, &first_smi, Label::kNear);
   __ cmpq(FieldOperand(first, HeapObject::kMapOffset), heap_number_map);
-  __ j(not_equal, on_not_smis);
+  __ j(not_equal, &maybe_undefined_first);
   // Convert HeapNumber to smi if possible.
   __ movsd(xmm0, FieldOperand(first, HeapNumber::kValueOffset));
   __ movq(scratch2, xmm0);
   __ cvttsd2siq(smi_result, xmm0);
   // Check if conversion was successful by converting back and
   // comparing to the original double's bits.
   __ cvtlsi2sd(xmm1, smi_result);
   __ movq(kScratchRegister, xmm1);
   __ cmpq(scratch2, kScratchRegister);
   __ j(not_equal, on_not_smis);
   __ Integer32ToSmi(first, smi_result);
 
+  __ bind(&first_done);
   __ JumpIfSmi(second, (on_success != NULL) ? on_success : &done);
   __ bind(&first_smi);
   __ AssertNotSmi(second);
   __ cmpq(FieldOperand(second, HeapObject::kMapOffset), heap_number_map);
-  __ j(not_equal, on_not_smis);
+  __ j(not_equal, &maybe_undefined_second);
   // Convert second to smi, if possible.
   __ movsd(xmm0, FieldOperand(second, HeapNumber::kValueOffset));
   __ movq(scratch2, xmm0);
   __ cvttsd2siq(smi_result, xmm0);
   __ cvtlsi2sd(xmm1, smi_result);
   __ movq(kScratchRegister, xmm1);
   __ cmpq(scratch2, kScratchRegister);
   __ j(not_equal, on_not_smis);
   __ Integer32ToSmi(second, smi_result);
   if (on_success != NULL) {
     __ jmp(on_success);
   } else {
-    __ bind(&done);
+    __ jmp(&done);
   }
+
+  __ bind(&maybe_undefined_first);
+  __ CompareRoot(first, Heap::kUndefinedValueRootIndex);
+  __ j(not_equal, on_not_smis);
+  __ xor_(first, first);
+  __ jmp(&first_done);
+
+  __ bind(&maybe_undefined_second);
+  __ CompareRoot(second, Heap::kUndefinedValueRootIndex);
+  __ j(not_equal, on_not_smis);
+  __ xor_(second, second);
+  if (on_success != NULL) {
+    __ jmp(on_success);
+  }
+  // Else: fall through.
+
+  __ bind(&done);
 }
 
 
 void MathPowStub::Generate(MacroAssembler* masm) {
   // Choose register conforming to calling convention (when bailing out).
 #ifdef _WIN64
   const Register exponent = rdx;
 #else
   const Register exponent = rdi;
 #endif
@@ skipping 1294 matching lines @@
 
 
 static int NegativeComparisonResult(Condition cc) {
   ASSERT(cc != equal);
   ASSERT((cc == less) || (cc == less_equal)
       || (cc == greater) || (cc == greater_equal));
   return (cc == greater || cc == greater_equal) ? LESS : GREATER;
 }
 
 
-void CompareStub::Generate(MacroAssembler* masm) {
-  ASSERT(lhs_.is(no_reg) && rhs_.is(no_reg));
+static void CheckInputType(MacroAssembler* masm,
+                           Register input,
+                           CompareIC::State expected,
+                           Label* fail) {
+  Label ok;
+  if (expected == CompareIC::SMI) {
+    __ JumpIfNotSmi(input, fail);
+  } else if (expected == CompareIC::HEAP_NUMBER) {
+    __ JumpIfSmi(input, &ok);
+    __ CompareMap(input, masm->isolate()->factory()->heap_number_map(), NULL);
+    __ j(not_equal, fail);
+  }
+  // We could be strict about symbol/string here, but as long as
+  // hydrogen doesn't care, the stub doesn't have to care either.
+  __ bind(&ok);
+}
 
+
+static void BranchIfNonSymbol(MacroAssembler* masm,
+                              Label* label,
+                              Register object,
+                              Register scratch) {
+  __ JumpIfSmi(object, label);
+  __ movq(scratch, FieldOperand(object, HeapObject::kMapOffset));
+  __ movzxbq(scratch,
+             FieldOperand(scratch, Map::kInstanceTypeOffset));
+  // Ensure that no non-strings have the symbol bit set.
+  STATIC_ASSERT(LAST_TYPE < kNotStringTag + kIsSymbolMask);
+  STATIC_ASSERT(kSymbolTag != 0);
+  __ testb(scratch, Immediate(kIsSymbolMask));
+  __ j(zero, label);
+}
+
+
+void ICCompareStub::GenerateGeneric(MacroAssembler* masm) {
   Label check_unequal_objects, done;
+  Condition cc = GetCondition();
   Factory* factory = masm->isolate()->factory();
 
-  // Compare two smis if required.
-  if (include_smi_compare_) {
-    Label non_smi, smi_done;
-    __ JumpIfNotBothSmi(rax, rdx, &non_smi);
-    __ subq(rdx, rax);
-    __ j(no_overflow, &smi_done);
-    __ not_(rdx);  // Correct sign in case of overflow. rdx cannot be 0 here.
-    __ bind(&smi_done);
-    __ movq(rax, rdx);
-    __ ret(0);
-    __ bind(&non_smi);
-  } else if (FLAG_debug_code) {
-    Label ok;
-    __ JumpIfNotSmi(rdx, &ok);
-    __ JumpIfNotSmi(rax, &ok);
-    __ Abort("CompareStub: smi operands");
-    __ bind(&ok);
-  }
+  Label miss;
+  CheckInputType(masm, rdx, left_, &miss);
+  CheckInputType(masm, rax, right_, &miss);
+
+  // Compare two smis.
+  Label non_smi, smi_done;
+  __ JumpIfNotBothSmi(rax, rdx, &non_smi);
+  __ subq(rdx, rax);
+  __ j(no_overflow, &smi_done);
+  __ not_(rdx);  // Correct sign in case of overflow. rdx cannot be 0 here.
+  __ bind(&smi_done);
+  __ movq(rax, rdx);
+  __ ret(0);
+  __ bind(&non_smi);
 
   // The compare stub returns a positive, negative, or zero 64-bit integer
   // value in rax, corresponding to result of comparing the two inputs.
   // NOTICE! This code is only reached after a smi-fast-case check, so
   // it is certain that at least one operand isn't a smi.
 
   // Two identical objects are equal unless they are both NaN or undefined.
   {
     Label not_identical;
     __ cmpq(rax, rdx);
     __ j(not_equal, &not_identical, Label::kNear);
 
-    if (cc_ != equal) {
+    if (cc != equal) {
       // Check for undefined.  undefined OP undefined is false even though
       // undefined == undefined.
       Label check_for_nan;
       __ CompareRoot(rdx, Heap::kUndefinedValueRootIndex);
       __ j(not_equal, &check_for_nan, Label::kNear);
-      __ Set(rax, NegativeComparisonResult(cc_));
+      __ Set(rax, NegativeComparisonResult(cc));
       __ ret(0);
       __ bind(&check_for_nan);
     }
 
     // Test for NaN. Sadly, we can't just compare to FACTORY->nan_value(),
     // so we do the second best thing - test it ourselves.
-    // Note: if cc_ != equal, never_nan_nan_ is not used.
-    // We cannot set rax to EQUAL until just before return because
-    // rax must be unchanged on jump to not_identical.
-    if (never_nan_nan_ && (cc_ == equal)) {
-      __ Set(rax, EQUAL);
-      __ ret(0);
-    } else {
-      Label heap_number;
-      // If it's not a heap number, then return equal for (in)equality operator.
-      __ Cmp(FieldOperand(rdx, HeapObject::kMapOffset),
-             factory->heap_number_map());
-      __ j(equal, &heap_number, Label::kNear);
-      if (cc_ != equal) {
-        // Call runtime on identical objects.  Otherwise return equal.
-        __ CmpObjectType(rax, FIRST_SPEC_OBJECT_TYPE, rcx);
-        __ j(above_equal, &not_identical, Label::kNear);
-      }
-      __ Set(rax, EQUAL);
-      __ ret(0);
+    Label heap_number;
+    // If it's not a heap number, then return equal for (in)equality operator.
+    __ Cmp(FieldOperand(rdx, HeapObject::kMapOffset),
+           factory->heap_number_map());
+    __ j(equal, &heap_number, Label::kNear);
+    if (cc != equal) {
+      // Call runtime on identical objects.  Otherwise return equal.
+      __ CmpObjectType(rax, FIRST_SPEC_OBJECT_TYPE, rcx);
+      __ j(above_equal, &not_identical, Label::kNear);
+    }
+    __ Set(rax, EQUAL);
+    __ ret(0);
 
-      __ bind(&heap_number);
-      // It is a heap number, so return  equal if it's not NaN.
-      // For NaN, return 1 for every condition except greater and
-      // greater-equal.  Return -1 for them, so the comparison yields
-      // false for all conditions except not-equal.
-      __ Set(rax, EQUAL);
-      __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
-      __ ucomisd(xmm0, xmm0);
-      __ setcc(parity_even, rax);
-      // rax is 0 for equal non-NaN heapnumbers, 1 for NaNs.
-      if (cc_ == greater_equal || cc_ == greater) {
-        __ neg(rax);
-      }
-      __ ret(0);
+    __ bind(&heap_number);
+    // It is a heap number, so return  equal if it's not NaN.
+    // For NaN, return 1 for every condition except greater and
+    // greater-equal.  Return -1 for them, so the comparison yields
+    // false for all conditions except not-equal.
+    __ Set(rax, EQUAL);
+    __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
+    __ ucomisd(xmm0, xmm0);
+    __ setcc(parity_even, rax);
+    // rax is 0 for equal non-NaN heapnumbers, 1 for NaNs.
+    if (cc == greater_equal || cc == greater) {
+      __ neg(rax);
     }
+    __ ret(0);
 
     __ bind(&not_identical);
   }
 
-  if (cc_ == equal) {  // Both strict and non-strict.
+  if (cc == equal) {  // Both strict and non-strict.
     Label slow;  // Fallthrough label.
 
     // If we're doing a strict equality comparison, we don't have to do
     // type conversion, so we generate code to do fast comparison for objects
     // and oddballs. Non-smi numbers and strings still go through the usual
     // slow-case code.
-    if (strict_) {
+    if (strict()) {
       // If either is a Smi (we know that not both are), then they can only
       // be equal if the other is a HeapNumber. If so, use the slow case.
       {
         Label not_smis;
         __ SelectNonSmi(rbx, rax, rdx, &not_smis);
 
         // Check if the non-smi operand is a heap number.
         __ Cmp(FieldOperand(rbx, HeapObject::kMapOffset),
                factory->heap_number_map());
         // If heap number, handle it in the slow case.
@@ skipping 31 matching lines @@
       // Check for oddballs: true, false, null, undefined.
       __ CmpInstanceType(rcx, ODDBALL_TYPE);
       __ j(equal, &return_not_equal);
 
       // Fall through to the general case.
     }
     __ bind(&slow);
   }
 
   // Generate the number comparison code.
-  if (include_number_compare_) {
-    Label non_number_comparison;
-    Label unordered;
-    FloatingPointHelper::LoadSSE2UnknownOperands(masm, &non_number_comparison);
-    __ xorl(rax, rax);
-    __ xorl(rcx, rcx);
-    __ ucomisd(xmm0, xmm1);
+  Label non_number_comparison;
+  Label unordered;
+  FloatingPointHelper::LoadSSE2UnknownOperands(masm, &non_number_comparison);
+  __ xorl(rax, rax);
+  __ xorl(rcx, rcx);
+  __ ucomisd(xmm0, xmm1);
 
-    // Don't base result on EFLAGS when a NaN is involved.
-    __ j(parity_even, &unordered, Label::kNear);
-    // Return a result of -1, 0, or 1, based on EFLAGS.
-    __ setcc(above, rax);
-    __ setcc(below, rcx);
-    __ subq(rax, rcx);
-    __ ret(0);
+  // Don't base result on EFLAGS when a NaN is involved.
+  __ j(parity_even, &unordered, Label::kNear);
+  // Return a result of -1, 0, or 1, based on EFLAGS.
+  __ setcc(above, rax);
+  __ setcc(below, rcx);
+  __ subq(rax, rcx);
+  __ ret(0);
 
-    // If one of the numbers was NaN, then the result is always false.
-    // The cc is never not-equal.
-    __ bind(&unordered);
-    ASSERT(cc_ != not_equal);
-    if (cc_ == less || cc_ == less_equal) {
-      __ Set(rax, 1);
-    } else {
-      __ Set(rax, -1);
-    }
-    __ ret(0);
+  // If one of the numbers was NaN, then the result is always false.
+  // The cc is never not-equal.
+  __ bind(&unordered);
+  ASSERT(cc != not_equal);
+  if (cc == less || cc == less_equal) {
+    __ Set(rax, 1);
+  } else {
+    __ Set(rax, -1);
+  }
+  __ ret(0);
 
-    // The number comparison code did not provide a valid result.
-    __ bind(&non_number_comparison);
-  }
+  // The number comparison code did not provide a valid result.
+  __ bind(&non_number_comparison);
 
   // Fast negative check for symbol-to-symbol equality.
   Label check_for_strings;
-  if (cc_ == equal) {
+  if (cc == equal) {
     BranchIfNonSymbol(masm, &check_for_strings, rax, kScratchRegister);
     BranchIfNonSymbol(masm, &check_for_strings, rdx, kScratchRegister);
 
     // We've already checked for object identity, so if both operands
     // are symbols they aren't equal. Register eax (not rax) already holds a
     // non-zero value, which indicates not equal, so just return.
     __ ret(0);
   }
 
   __ bind(&check_for_strings);
 
   __ JumpIfNotBothSequentialAsciiStrings(
       rdx, rax, rcx, rbx, &check_unequal_objects);
 
   // Inline comparison of ASCII strings.
-  if (cc_ == equal) {
+  if (cc == equal) {
     StringCompareStub::GenerateFlatAsciiStringEquals(masm,
                                                      rdx,
                                                      rax,
                                                      rcx,
                                                      rbx);
   } else {
     StringCompareStub::GenerateCompareFlatAsciiStrings(masm,
                                                        rdx,
                                                        rax,
                                                        rcx,
                                                        rbx,
                                                        rdi,
                                                        r8);
   }
 
 #ifdef DEBUG
   __ Abort("Unexpected fall-through from string comparison");
 #endif
 
   __ bind(&check_unequal_objects);
-  if (cc_ == equal && !strict_) {
+  if (cc == equal && !strict()) {
     // Not strict equality.  Objects are unequal if
     // they are both JSObjects and not undetectable,
     // and their pointers are different.
     Label not_both_objects, return_unequal;
     // At most one is a smi, so we can test for smi by adding the two.
     // A smi plus a heap object has the low bit set, a heap object plus
     // a heap object has the low bit clear.
     STATIC_ASSERT(kSmiTag == 0);
     STATIC_ASSERT(kSmiTagMask == 1);
     __ lea(rcx, Operand(rax, rdx, times_1, 0));
@@ skipping 19 matching lines @@
     __ bind(&not_both_objects);
   }
 
   // Push arguments below the return address to prepare jump to builtin.
   __ pop(rcx);
   __ push(rdx);
   __ push(rax);
 
   // Figure out which native to call and setup the arguments.
   Builtins::JavaScript builtin;
-  if (cc_ == equal) {
-    builtin = strict_ ? Builtins::STRICT_EQUALS : Builtins::EQUALS;
+  if (cc == equal) {
+    builtin = strict() ? Builtins::STRICT_EQUALS : Builtins::EQUALS;
   } else {
     builtin = Builtins::COMPARE;
-    __ Push(Smi::FromInt(NegativeComparisonResult(cc_)));
+    __ Push(Smi::FromInt(NegativeComparisonResult(cc)));
   }
 
   // Restore return address on the stack.
   __ push(rcx);
 
   // Call the native; it returns -1 (less), 0 (equal), or 1 (greater)
   // tagged as a small integer.
   __ InvokeBuiltin(builtin, JUMP_FUNCTION);
+
+  __ bind(&miss);
+  GenerateMiss(masm);
 }
 
 
-void CompareStub::BranchIfNonSymbol(MacroAssembler* masm,
-                                    Label* label,
-                                    Register object,
-                                    Register scratch) {
-  __ JumpIfSmi(object, label);
-  __ movq(scratch, FieldOperand(object, HeapObject::kMapOffset));
-  __ movzxbq(scratch,
-             FieldOperand(scratch, Map::kInstanceTypeOffset));
-  // Ensure that no non-strings have the symbol bit set.
-  STATIC_ASSERT(LAST_TYPE < kNotStringTag + kIsSymbolMask);
-  STATIC_ASSERT(kSymbolTag != 0);
-  __ testb(scratch, Immediate(kIsSymbolMask));
-  __ j(zero, label);
-}
-
-
 void StackCheckStub::Generate(MacroAssembler* masm) {
   __ TailCallRuntime(Runtime::kStackGuard, 0, 1);
 }
 
 
 void InterruptStub::Generate(MacroAssembler* masm) {
   __ TailCallRuntime(Runtime::kInterrupt, 0, 1);
 }
 
 
@@ skipping 734 matching lines @@
 }
 
 
 // Passing arguments in registers is not supported.
 Register InstanceofStub::left() { return no_reg; }
 
 
 Register InstanceofStub::right() { return no_reg; }
 
 
-int CompareStub::MinorKey() {
-  // Encode the three parameters in a unique 16 bit value. To avoid duplicate
-  // stubs the never NaN NaN condition is only taken into account if the
-  // condition is equals.
-  ASSERT(static_cast<unsigned>(cc_) < (1 << 12));
-  ASSERT(lhs_.is(no_reg) && rhs_.is(no_reg));
-  return ConditionField::encode(static_cast<unsigned>(cc_))
-         | RegisterField::encode(false)    // lhs_ and rhs_ are not used
-         | StrictField::encode(strict_)
-         | NeverNanNanField::encode(cc_ == equal ? never_nan_nan_ : false)
-         | IncludeNumberCompareField::encode(include_number_compare_)
-         | IncludeSmiCompareField::encode(include_smi_compare_);
-}
-
-
-// Unfortunately you have to run without snapshots to see most of these
-// names in the profile since most compare stubs end up in the snapshot.
-void CompareStub::PrintName(StringStream* stream) {
-  ASSERT(lhs_.is(no_reg) && rhs_.is(no_reg));
-  const char* cc_name;
-  switch (cc_) {
-    case less: cc_name = "LT"; break;
-    case greater: cc_name = "GT"; break;
-    case less_equal: cc_name = "LE"; break;
-    case greater_equal: cc_name = "GE"; break;
-    case equal: cc_name = "EQ"; break;
-    case not_equal: cc_name = "NE"; break;
-    default: cc_name = "UnknownCondition"; break;
-  }
-  bool is_equality = cc_ == equal || cc_ == not_equal;
-  stream->Add("CompareStub_%s", cc_name);
-  if (strict_ && is_equality) stream->Add("_STRICT");
-  if (never_nan_nan_ && is_equality) stream->Add("_NO_NAN");
-  if (!include_number_compare_) stream->Add("_NO_NUMBER");
-  if (!include_smi_compare_) stream->Add("_NO_SMI");
-}
-
-
 // -------------------------------------------------------------------------
 // StringCharCodeAtGenerator
 
 void StringCharCodeAtGenerator::GenerateFast(MacroAssembler* masm) {
   Label flat_string;
   Label ascii_string;
   Label got_char_code;
   Label sliced_string;
 
   // If the receiver is a smi trigger the non-string case.
@@ skipping 1089 matching lines @@
   GenerateCompareFlatAsciiStrings(masm, rdx, rax, rcx, rbx, rdi, r8);
 
   // Call the runtime; it returns -1 (less), 0 (equal), or 1 (greater)
   // tagged as a small integer.
   __ bind(&runtime);
   __ TailCallRuntime(Runtime::kStringCompare, 2, 1);
 }
 
 
 void ICCompareStub::GenerateSmis(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::SMIS);
+  ASSERT(state_ == CompareIC::SMI);
   Label miss;
   __ JumpIfNotBothSmi(rdx, rax, &miss, Label::kNear);
 
   if (GetCondition() == equal) {
     // For equality we do not care about the sign of the result.
     __ subq(rax, rdx);
   } else {
     Label done;
     __ subq(rdx, rax);
     __ j(no_overflow, &done, Label::kNear);
     // Correct sign of result in case of overflow.
     __ SmiNot(rdx, rdx);
     __ bind(&done);
     __ movq(rax, rdx);
   }
   __ ret(0);
 
   __ bind(&miss);
   GenerateMiss(masm);
 }
 
 
 void ICCompareStub::GenerateHeapNumbers(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::HEAP_NUMBERS);
+  ASSERT(state_ == CompareIC::HEAP_NUMBER);
 
   Label generic_stub;
   Label unordered, maybe_undefined1, maybe_undefined2;
   Label miss;
-  Condition either_smi = masm->CheckEitherSmi(rax, rdx);
-  __ j(either_smi, &generic_stub, Label::kNear);
 
-  __ CmpObjectType(rax, HEAP_NUMBER_TYPE, rcx);
+  if (left_ == CompareIC::SMI) {
+    __ JumpIfNotSmi(rdx, &miss);
+  }
+  if (right_ == CompareIC::SMI) {
+    __ JumpIfNotSmi(rax, &miss);
+  }
+
+  // Load left and right operand.
+  Label done, left, left_smi, right_smi;
+  __ JumpIfSmi(rax, &right_smi, Label::kNear);
+  __ CompareMap(rax, masm->isolate()->factory()->heap_number_map(), NULL);
   __ j(not_equal, &maybe_undefined1, Label::kNear);
-  __ CmpObjectType(rdx, HEAP_NUMBER_TYPE, rcx);
+  __ movsd(xmm1, FieldOperand(rax, HeapNumber::kValueOffset));
+  __ jmp(&left, Label::kNear);
+  __ bind(&right_smi);
+  __ SmiToInteger32(rcx, rax);  // Can't clobber rax yet.
+  __ cvtlsi2sd(xmm1, rcx);
+
+  __ bind(&left);
+  __ JumpIfSmi(rdx, &left_smi, Label::kNear);
+  __ CompareMap(rdx, masm->isolate()->factory()->heap_number_map(), NULL);
   __ j(not_equal, &maybe_undefined2, Label::kNear);
+  __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
+  __ jmp(&done);
+  __ bind(&left_smi);
+  __ SmiToInteger32(rcx, rdx);  // Can't clobber rdx yet.
+  __ cvtlsi2sd(xmm0, rcx);
 
-  // Load left and right operand
-  __ movsd(xmm0, FieldOperand(rdx, HeapNumber::kValueOffset));
-  __ movsd(xmm1, FieldOperand(rax, HeapNumber::kValueOffset));
-
+  __ bind(&done);
   // Compare operands
   __ ucomisd(xmm0, xmm1);
 
   // Don't base result on EFLAGS when a NaN is involved.
   __ j(parity_even, &unordered, Label::kNear);
 
   // Return a result of -1, 0, or 1, based on EFLAGS.
   // Performing mov, because xor would destroy the flag register.
   __ movl(rax, Immediate(0));
   __ movl(rcx, Immediate(0));
   __ setcc(above, rax);  // Add one to zero if carry clear and not equal.
   __ sbbq(rax, rcx);  // Subtract one if below (aka. carry set).
   __ ret(0);
 
   __ bind(&unordered);
-  CompareStub stub(GetCondition(), strict(), NO_COMPARE_FLAGS);
   __ bind(&generic_stub);
+  ICCompareStub stub(op_, CompareIC::GENERIC, CompareIC::GENERIC,
+                     CompareIC::GENERIC);
   __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET);
 
   __ bind(&maybe_undefined1);
   if (Token::IsOrderedRelationalCompareOp(op_)) {
     __ Cmp(rax, masm->isolate()->factory()->undefined_value());
     __ j(not_equal, &miss);
+    __ JumpIfSmi(rdx, &unordered);
     __ CmpObjectType(rdx, HEAP_NUMBER_TYPE, rcx);
     __ j(not_equal, &maybe_undefined2, Label::kNear);
     __ jmp(&unordered);
   }
 
   __ bind(&maybe_undefined2);
   if (Token::IsOrderedRelationalCompareOp(op_)) {
     __ Cmp(rdx, masm->isolate()->factory()->undefined_value());
     __ j(equal, &unordered);
   }
 
   __ bind(&miss);
   GenerateMiss(masm);
 }
 
 
 void ICCompareStub::GenerateSymbols(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::SYMBOLS);
+  ASSERT(state_ == CompareIC::SYMBOL);
   ASSERT(GetCondition() == equal);
 
   // Registers containing left and right operands respectively.
   Register left = rdx;
   Register right = rax;
   Register tmp1 = rcx;
   Register tmp2 = rbx;
 
   // Check that both operands are heap objects.
   Label miss;
@@ skipping 22 matching lines @@
   __ Move(rax, Smi::FromInt(EQUAL));
   __ bind(&done);
   __ ret(0);
 
   __ bind(&miss);
   GenerateMiss(masm);
 }
 
 
 void ICCompareStub::GenerateStrings(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::STRINGS);
+  ASSERT(state_ == CompareIC::STRING);
   Label miss;
 
   bool equality = Token::IsEqualityOp(op_);
 
   // Registers containing left and right operands respectively.
   Register left = rdx;
   Register right = rax;
   Register tmp1 = rcx;
   Register tmp2 = rbx;
   Register tmp3 = rdi;
@@ skipping 65 matching lines @@
   } else {
     __ TailCallRuntime(Runtime::kStringCompare, 2, 1);
   }
 
   __ bind(&miss);
   GenerateMiss(masm);
 }
 
 
 void ICCompareStub::GenerateObjects(MacroAssembler* masm) {
-  ASSERT(state_ == CompareIC::OBJECTS);
+  ASSERT(state_ == CompareIC::OBJECT);
   Label miss;
   Condition either_smi = masm->CheckEitherSmi(rdx, rax);
   __ j(either_smi, &miss, Label::kNear);
 
   __ CmpObjectType(rax, JS_OBJECT_TYPE, rcx);
   __ j(not_equal, &miss, Label::kNear);
   __ CmpObjectType(rdx, JS_OBJECT_TYPE, rcx);
   __ j(not_equal, &miss, Label::kNear);
 
   ASSERT(GetCondition() == equal);
@@ skipping 707 matching lines @@
 #endif
 
   __ Ret();
 }
 
 #undef __
 
 } }  // namespace v8::internal
 
 #endif  // V8_TARGET_ARCH_X64