X64: Added inline keyed load/store and a bunch of other missing functions.

src/x64/ic-x64.cc

 // Copyright 2009 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 141 matching lines @@
   __ j(zero, &done);
   // Check if the object has been loaded.
   __ movq(kScratchRegister, FieldOperand(value, JSFunction::kMapOffset));
   __ testb(FieldOperand(kScratchRegister, Map::kBitField2Offset),
            Immediate(1 << Map::kNeedsLoading));
   __ j(not_zero, miss);
   __ bind(&done);
 }
 
 
+// One byte opcode for test eax,0xXXXXXXXX.
+static const byte kTestEaxByte = 0xA9;
+
+
+static bool PatchInlinedMapCheck(Address address, Object* map) {
+  // Arguments are address of start of call sequence that called
+  // the IC,
+  Address test_instruction_address =
+      address + Assembler::kTargetAddrToReturnAddrDist;
+  // The keyed load has a fast inlined case if the IC call instruction
+  // is immediately followed by a test instruction.
+  if (*test_instruction_address != kTestEaxByte) return false;
+
+  // Fetch the offset from the test instruction to the map compare
+  // instructions (starting with the 64-bit immediate mov of the map
+  // address). This offset is stored in the last 4 bytes of the 5
+  // byte test instruction.
+  Address delta_address = test_instruction_address + 1;
+  int delta = *reinterpret_cast<int*>(delta_address);
+  // Compute the map address.  The map address is in the last 8 bytes
+  // of the 10-byte immediate mov instruction (incl. REX prefix), so we add 2
+  // to the offset to get the map address.
+  Address map_address = test_instruction_address + delta + 2;
+  // Patch the map check.
+  *(reinterpret_cast<Object**>(map_address)) = map;
+  return true;
+}
+
+
+bool KeyedLoadIC::PatchInlinedLoad(Address address, Object* map) {
+  return PatchInlinedMapCheck(address, map);
+}
+
+
+bool KeyedStoreIC::PatchInlinedStore(Address address, Object* map) {
+  return PatchInlinedMapCheck(address, map);
+}
+
+
 void KeyedLoadIC::ClearInlinedVersion(Address address) {
-  // TODO(X64): Implement this when LoadIC is enabled.
-}
-
-void KeyedStoreIC::ClearInlinedVersion(Address address) {
-  // TODO(X64): Implement this when LoadIC is enabled.
-}
-
-void KeyedStoreIC::RestoreInlinedVersion(Address address) {
-  UNIMPLEMENTED();
+  // Insert null as the map to check for to make sure the map check fails
+  // sending control flow to the IC instead of the inlined version.
+  PatchInlinedLoad(address, Heap::null_value());
 }
 
 
+void KeyedStoreIC::ClearInlinedVersion(Address address) {
+  // Insert null as the elements map to check for.  This will make
+  // sure that the elements fast-case map check fails so that control
+  // flows to the IC instead of the inlined version.
+  PatchInlinedStore(address, Heap::null_value());
+}
+
+
+void KeyedStoreIC::RestoreInlinedVersion(Address address) {
+  // Restore the fast-case elements map check so that the inlined
+  // version can be used again.
+  PatchInlinedStore(address, Heap::fixed_array_map());
+}
+
+
 void KeyedLoadIC::Generate(MacroAssembler* masm,
                            ExternalReference const& f) {
   // ----------- S t a t e -------------
   //  -- rsp[0]  : return address
   //  -- rsp[8]  : name
   //  -- rsp[16] : receiver
   // -----------------------------------
 
   __ movq(rax, Operand(rsp, kPointerSize));
   __ movq(rcx, Operand(rsp, 2 * kPointerSize));
@@ skipping 118 matching lines @@
 void KeyedLoadIC::GenerateMiss(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- rsp[0] : return address
   //  -- rsp[8] : name
   //  -- rsp[16] : receiver
   // -----------------------------------
   Generate(masm, ExternalReference(IC_Utility(kKeyedLoadIC_Miss)));
 }
 
 
-bool KeyedLoadIC::PatchInlinedLoad(Address address, Object* map) {
-  // Never patch the map in the map check, so the check always fails.
-  return false;
-}
-
-
-bool KeyedStoreIC::PatchInlinedStore(Address address, Object* map) {
-  // Never patch the map in the map check, so the check always fails.
-  return false;
-}
-
-
 void KeyedStoreIC::Generate(MacroAssembler* masm, ExternalReference const& f) {
   // ----------- S t a t e -------------
   //  -- rax     : value
   //  -- rsp[0]  : return address
   //  -- rsp[8]  : key
   //  -- rsp[16] : receiver
   // -----------------------------------
 
   __ pop(rcx);
   __ push(Operand(rsp, 1 * kPointerSize));  // receiver
@@ skipping 197 matching lines @@
   Generate(masm, argc, ExternalReference(IC_Utility(kCallIC_Miss)));
 }
 
 
 // The offset from the inlined patch site to the start of the
 // inlined load instruction.
 const int LoadIC::kOffsetToLoadInstruction = 20;
 
 
 void LoadIC::ClearInlinedVersion(Address address) {
-  // TODO(X64): Implement this when LoadIC is enabled.
+  // Reset the map check of the inlined inobject property load (if
+  // present) to guarantee failure by holding an invalid map (the null
+  // value).  The offset can be patched to anything.
+  PatchInlinedLoad(address, Heap::null_value(), kMaxInt);
 }
 
 
 void LoadIC::Generate(MacroAssembler* masm, ExternalReference const& f) {
   // ----------- S t a t e -------------
   //  -- rcx    : name
   //  -- rsp[0] : return address
   //  -- rsp[8] : receiver
   // -----------------------------------
 
@@ skipping 45 matching lines @@
   //  -- rsp[8] : receiver
   // -----------------------------------
 
   Generate(masm, ExternalReference(IC_Utility(kLoadIC_Miss)));
 }
 
 void LoadIC::GenerateNormal(MacroAssembler* masm) {
   Generate(masm, ExternalReference(IC_Utility(kLoadIC_Miss)));
 }
 
+
 void LoadIC::GenerateStringLength(MacroAssembler* masm) {
   Generate(masm, ExternalReference(IC_Utility(kLoadIC_Miss)));
 }
 
-bool LoadIC::PatchInlinedLoad(Address address, Object* map, int index) {
-  // TODO(X64): Implement this function.  Until then, the code is not patched.
-  return false;
+
+bool LoadIC::PatchInlinedLoad(Address address, Object* map, int offset) {
+  // The address of the instruction following the call.
+  Address test_instruction_address =
+      address + Assembler::kTargetAddrToReturnAddrDist;
+  // If the instruction following the call is not a test eax, nothing
+  // was inlined.
+  if (*test_instruction_address != kTestEaxByte) return false;
+
+  Address delta_address = test_instruction_address + 1;
+  // The delta to the start of the map check instruction.
+  int delta = *reinterpret_cast<int*>(delta_address);
+
+  // The map address is the last 8 bytes of the 10-byte
+  // immediate move instruction, so we add 2 to get the
+  // offset to the last 8 bytes.
+  Address map_address = test_instruction_address + delta + 2;
+  *(reinterpret_cast<Object**>(map_address)) = map;
+
+  // The offset is in the 32-bit displacement of a seven byte
+  // memory-to-register move instruction (REX.W 0x88 ModR/M disp32),
+  // so we add 3 to get the offset of the displacement.
+  Address offset_address =
+      test_instruction_address + delta + kOffsetToLoadInstruction + 3;
+  *reinterpret_cast<int*>(offset_address) = offset - kHeapObjectTag;
+  return true;
 }
 
 void StoreIC::Generate(MacroAssembler* masm, ExternalReference const& f) {
   // ----------- S t a t e -------------
   //  -- rax    : value
   //  -- rcx    : name
   //  -- rsp[0] : return address
   //  -- rsp[8] : receiver
   // -----------------------------------
   __ pop(rbx);
@@ skipping 27 matching lines @@
 
 void StoreIC::GenerateMegamorphic(MacroAssembler* masm) {
   Generate(masm, ExternalReference(IC_Utility(kStoreIC_Miss)));
 }
 
 
 #undef __
 
 
 } }  // namespace v8::internal