Port KeyedCallIC implementation to x64 and ARM....

src/x64/ic-x64.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 39 matching lines @@
 // must always call a backup property load that is complete.
 // This function is safe to call if the receiver has fast properties,
 // or if name is not a symbol, and will jump to the miss_label in that case.
 static void GenerateDictionaryLoad(MacroAssembler* masm,
                                    Label* miss_label,
                                    Register r0,
                                    Register r1,
                                    Register r2,
                                    Register name,
                                    Register r4,
+                                   Register result,
                                    DictionaryCheck check_dictionary) {
   // Register use:
   //
-  // r0   - used to hold the property dictionary.
+  // r0   - used to hold the property dictionary and is unchanged.
   //
-  // r1   - initially the receiver.
-  //      - unchanged on any jump to miss_label.
-  //      - holds the result on exit.
+  // r1   - used to hold the receiver and is unchanged.
   //
   // r2   - used to hold the capacity of the property dictionary.
   //
   // name - holds the name of the property and is unchanged.
+  //
   // r4   - used to hold the index into the property dictionary.
+  //
+  // result - holds the result on exit if the load succeeded.
 
   Label done;
 
   // Check for the absence of an interceptor.
   // Load the map into r0.
   __ movq(r0, FieldOperand(r1, JSObject::kMapOffset));
 
   // Bail out if the receiver has a named interceptor.
   __ testl(FieldOperand(r0, Map::kBitFieldOffset),
            Immediate(1 << Map::kHasNamedInterceptor));
@@ skipping 58 matching lines @@
 
   // Check that the value is a normal property.
   __ bind(&done);
   const int kDetailsOffset = kElementsStartOffset + 2 * kPointerSize;
   __ Test(Operand(r0, r4, times_pointer_size, kDetailsOffset - kHeapObjectTag),
           Smi::FromInt(PropertyDetails::TypeField::mask()));
   __ j(not_zero, miss_label);
 
   // Get the value at the masked, scaled index.
   const int kValueOffset = kElementsStartOffset + kPointerSize;
-  __ movq(r1,
+  __ movq(result,
           Operand(r0, r4, times_pointer_size, kValueOffset - kHeapObjectTag));
 }
 
 
 static void GenerateNumberDictionaryLoad(MacroAssembler* masm,
                                          Label* miss,
                                          Register elements,
                                          Register key,
                                          Register r0,
                                          Register r1,
-                                         Register r2) {
+                                         Register r2,
+                                         Register result) {
   // Register use:
   //
-  // elements - holds the slow-case elements of the receiver and is unchanged.
+  // elements - holds the slow-case elements of the receiver on entry.
+  //            Unchanged unless 'result' is the same register.
   //
-  // key      - holds the smi key on entry and is unchanged if a branch is
-  //            performed to the miss label.
-  //            Holds the result on exit if the load succeeded.
+  // key      - holds the smi key on entry.
+  //            Unchanged unless 'result' is the same register.
   //
   // Scratch registers:
   //
   // r0 - holds the untagged key on entry and holds the hash once computed.
   //
   // r1 - used to hold the capacity mask of the dictionary
   //
   // r2 - used for the index into the dictionary.
+  //
+  // result - holds the result on exit if the load succeeded.
+  //          Allowed to be the same as 'key' or 'result'.
+  //          Unchanged on bailout so 'key' or 'result' can be used
+  //          in further computation.
+
   Label done;
 
   // Compute the hash code from the untagged key.  This must be kept in sync
   // with ComputeIntegerHash in utils.h.
   //
   // hash = ~hash + (hash << 15);
   __ movl(r1, r0);
   __ notl(r0);
   __ shll(r1, Immediate(15));
   __ addl(r0, r1);
@@ skipping 51 matching lines @@
   const int kDetailsOffset =
       NumberDictionary::kElementsStartOffset + 2 * kPointerSize;
   ASSERT_EQ(NORMAL, 0);
   __ Test(FieldOperand(elements, r2, times_pointer_size, kDetailsOffset),
           Smi::FromInt(PropertyDetails::TypeField::mask()));
   __ j(not_zero, miss);
 
   // Get the value at the masked, scaled index.
   const int kValueOffset =
       NumberDictionary::kElementsStartOffset + kPointerSize;
-  __ movq(key, FieldOperand(elements, r2, times_pointer_size, kValueOffset));
+  __ movq(result, FieldOperand(elements, r2, times_pointer_size, kValueOffset));
 }
 
 
 // One byte opcode for test eax,0xXXXXXXXX.
 static const byte kTestEaxByte = 0xA9;
 
 
 static bool PatchInlinedMapCheck(Address address, Object* map) {
   // Arguments are address of start of call sequence that called
   // the IC,
@@ skipping 79 matching lines @@
   __ pop(rbx);
   __ push(rdx);  // receiver
   __ push(rax);  // name
   __ push(rbx);  // return address
 
   // Perform tail call to the entry.
   __ TailCallRuntime(Runtime::kKeyedGetProperty, 2, 1);
 }
 
 
+// Checks the receiver for special cases (value type, slow case bits).
+// Falls through for regular JS object.
+static void GenerateKeyedLoadReceiverCheck(MacroAssembler* masm,
+                                           Register receiver,
+                                           Register map,
+                                           Label* slow) {
+  // Register use:
+  //   receiver - holds the receiver and is unchanged.
+  // Scratch registers:
+  //   map - used to hold the map of the receiver.
+
+  // Check that the object isn't a smi.
+  __ JumpIfSmi(receiver, slow);
+
+  // Check that the object is some kind of JS object EXCEPT JS Value type.
+  // In the case that the object is a value-wrapper object,
+  // we enter the runtime system to make sure that indexing
+  // into string objects work as intended.
+  ASSERT(JS_OBJECT_TYPE > JS_VALUE_TYPE);
+  __ CmpObjectType(receiver, JS_OBJECT_TYPE, map);
+  __ j(below, slow);
+
+  // Check bit field.
+  __ testb(FieldOperand(map, Map::kBitFieldOffset),
+           Immediate(KeyedLoadIC::kSlowCaseBitFieldMask));
+  __ j(not_zero, slow);
+}
+
+
+// Loads an indexed element from a fast case array.
+static void GenerateFastArrayLoad(MacroAssembler* masm,
+                                  Register receiver,
+                                  Register key,
+                                  Register elements,
+                                  Register scratch,
+                                  Register result,
+                                  Label* not_fast_array,
+                                  Label* out_of_range) {
+  // Register use:
+  //
+  // receiver - holds the receiver on entry.
+  //            Unchanged unless 'result' is the same register.
+  //
+  // key      - holds the smi key on entry.
+  //            Unchanged unless 'result' is the same register.
+  //
+  // elements - holds the elements of the receiver on exit.
+  //
+  // result   - holds the result on exit if the load succeeded.
+  //            Allowed to be the the same as 'receiver' or 'key'.
+  //            Unchanged on bailout so 'receiver' and 'key' can be safely
+  //            used by further computation.
+  //
+  // Scratch registers:
+  //
+  //   scratch - used to hold elements of the receiver and the loaded value.
+
+  __ movq(elements, FieldOperand(receiver, JSObject::kElementsOffset));
+  // Check that the object is in fast mode (not dictionary).
+  __ CompareRoot(FieldOperand(elements, HeapObject::kMapOffset),
+                 Heap::kFixedArrayMapRootIndex);
+  __ j(not_equal, not_fast_array);
+  // Check that the key (index) is within bounds.
+  __ SmiCompare(key, FieldOperand(elements, FixedArray::kLengthOffset));
+  // Unsigned comparison rejects negative indices.
+  __ j(above_equal, out_of_range);
+  // Fast case: Do the load.
+  SmiIndex index = masm->SmiToIndex(scratch, key, kPointerSizeLog2);
+  __ movq(scratch, FieldOperand(elements,
+                                index.reg,
+                                index.scale,
+                                FixedArray::kHeaderSize));
+  __ CompareRoot(scratch, Heap::kTheHoleValueRootIndex);
+  // In case the loaded value is the_hole we have to consult GetProperty
+  // to ensure the prototype chain is searched.
+  __ j(equal, out_of_range);
+  if (!result.is(scratch)) {
+    __ movq(result, scratch);
+  }
+}
+
+
+// Checks whether a key is an array index string or a symbol string.
+// Falls through if the key is a symbol.
+static void GenerateKeyStringCheck(MacroAssembler* masm,
+                                   Register key,
+                                   Register map,
+                                   Register hash,
+                                   Label* index_string,
+                                   Label* not_symbol) {
+  // Register use:
+  //   key - holds the key and is unchanged. Assumed to be non-smi.
+  // Scratch registers:
+  //   map - used to hold the map of the key.
+  //   hash - used to hold the hash of the key.
+  __ CmpObjectType(key, FIRST_NONSTRING_TYPE, map);
+  __ j(above_equal, not_symbol);
+  // Is the string an array index, with cached numeric value?
+  __ movl(hash, FieldOperand(key, String::kHashFieldOffset));
+  __ testl(hash, Immediate(String::kContainsCachedArrayIndexMask));
+  __ j(zero, index_string);  // The value in hash is used at jump target.
+
+  // Is the string a symbol?
+  ASSERT(kSymbolTag != 0);
+  __ testb(FieldOperand(map, Map::kInstanceTypeOffset),
+           Immediate(kIsSymbolMask));
+  __ j(zero, not_symbol);
+}
+
+
+// Picks out an array index from the hash field.
+static void GenerateIndexFromHash(MacroAssembler* masm,
+                                  Register key,
+                                  Register hash) {
+  // Register use:
+  //   key - holds the overwritten key on exit.
+  //   hash - holds the key's hash. Clobbered.
+
+  // The assert checks that the constants for the maximum number of digits
+  // for an array index cached in the hash field and the number of bits
+  // reserved for it does not conflict.
+  ASSERT(TenToThe(String::kMaxCachedArrayIndexLength) <
+         (1 << String::kArrayIndexValueBits));
+  // We want the smi-tagged index in key. Even if we subsequently go to
+  // the slow case, converting the key to a smi is always valid.
+  // key: string key
+  // hash: key's hash field, including its array index value.
+  __ and_(hash, Immediate(String::kArrayIndexValueMask));
+  __ shr(hash, Immediate(String::kHashShift));
+  // Here we actually clobber the key which will be used if calling into
+  // runtime later. However as the new key is the numeric value of a string key
+  // there is no difference in using either key.
+  __ Integer32ToSmi(key, hash);
+}
+
+
 void KeyedLoadIC::GenerateGeneric(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- rax    : key
   //  -- rdx    : receiver
   //  -- rsp[0] : return address
   // -----------------------------------
   Label slow, check_string, index_smi, index_string;
   Label check_pixel_array, probe_dictionary, check_number_dictionary;
 
-  // Check that the object isn't a smi.
-  __ JumpIfSmi(rdx, &slow);
-
-  // Check that the object is some kind of JS object EXCEPT JS Value type.
-  // In the case that the object is a value-wrapper object,
-  // we enter the runtime system to make sure that indexing
-  // into string objects work as intended.
-  ASSERT(JS_OBJECT_TYPE > JS_VALUE_TYPE);
-  __ CmpObjectType(rdx, JS_OBJECT_TYPE, rcx);
-  __ j(below, &slow);
-
-  // Check bit field.
-  __ testb(FieldOperand(rcx, Map::kBitFieldOffset),
-           Immediate(kSlowCaseBitFieldMask));
-  __ j(not_zero, &slow);
+  GenerateKeyedLoadReceiverCheck(masm, rdx, rcx, &slow);
 
   // Check that the key is a smi.
   __ JumpIfNotSmi(rax, &check_string);
   __ bind(&index_smi);
   // Now the key is known to be a smi. This place is also jumped to from below
   // where a numeric string is converted to a smi.
-  __ movq(rcx, FieldOperand(rdx, JSObject::kElementsOffset));
-  // Check that the object is in fast mode (not dictionary).
-  __ CompareRoot(FieldOperand(rcx, HeapObject::kMapOffset),
-                 Heap::kFixedArrayMapRootIndex);
-  __ j(not_equal, &check_pixel_array);
-  // Check that the key (index) is within bounds.
-  __ SmiCompare(rax, FieldOperand(rcx, FixedArray::kLengthOffset));
-  __ j(above_equal, &slow);  // Unsigned comparison rejects negative indices.
-  // Fast case: Do the load.
-  SmiIndex index = masm->SmiToIndex(rbx, rax, kPointerSizeLog2);
-  __ movq(rbx, FieldOperand(rcx,
-                            index.reg,
-                            index.scale,
-                            FixedArray::kHeaderSize));
-  __ CompareRoot(rbx, Heap::kTheHoleValueRootIndex);
-  // In case the loaded value is the_hole we have to consult GetProperty
-  // to ensure the prototype chain is searched.
-  __ j(equal, &slow);
-  __ movq(rax, rbx);
+
+  GenerateFastArrayLoad(masm,
+                        rdx,
+                        rax,
+                        rcx,
+                        rbx,
+                        rax,
+                        &check_pixel_array,
+                        &slow);
   __ IncrementCounter(&Counters::keyed_load_generic_smi, 1);
   __ ret(0);
 
   __ bind(&check_pixel_array);
   // Check whether the elements object is a pixel array.
   // rdx: receiver
   // rax: key
   // rcx: elements array
   __ SmiToInteger32(rbx, rax);  // Used on both directions of next branch.
   __ CompareRoot(FieldOperand(rcx, HeapObject::kMapOffset),
                  Heap::kPixelArrayMapRootIndex);
   __ j(not_equal, &check_number_dictionary);
   __ cmpl(rbx, FieldOperand(rcx, PixelArray::kLengthOffset));
   __ j(above_equal, &slow);
   __ movq(rax, FieldOperand(rcx, PixelArray::kExternalPointerOffset));
   __ movzxbq(rax, Operand(rax, rbx, times_1, 0));
   __ Integer32ToSmi(rax, rax);
   __ ret(0);
 
   __ bind(&check_number_dictionary);
   // Check whether the elements is a number dictionary.
   // rdx: receiver
   // rax: key
   // rbx: key as untagged int32
   // rcx: elements
   __ CompareRoot(FieldOperand(rcx, HeapObject::kMapOffset),
                  Heap::kHashTableMapRootIndex);
   __ j(not_equal, &slow);
-  GenerateNumberDictionaryLoad(masm, &slow, rcx, rax, rbx, r9, rdi);
+  GenerateNumberDictionaryLoad(masm, &slow, rcx, rax, rbx, r9, rdi, rax);
   __ ret(0);
 
   __ bind(&slow);
   // Slow case: Jump to runtime.
   // rdx: receiver
   // rax: key
   __ IncrementCounter(&Counters::keyed_load_generic_slow, 1);
   GenerateRuntimeGetProperty(masm);
 
   __ bind(&check_string);
-  // The key is not a smi.
-  // Is it a string?
-  // rdx: receiver
-  // rax: key
-  __ CmpObjectType(rax, FIRST_NONSTRING_TYPE, rcx);
-  __ j(above_equal, &slow);
-  // Is the string an array index, with cached numeric value?
-  __ movl(rbx, FieldOperand(rax, String::kHashFieldOffset));
-  __ testl(rbx, Immediate(String::kContainsCachedArrayIndexMask));
-  __ j(zero, &index_string);  // The value in rbx is used at jump target.
-
-  // Is the string a symbol?
-  ASSERT(kSymbolTag != 0);
-  __ testb(FieldOperand(rcx, Map::kInstanceTypeOffset),
-           Immediate(kIsSymbolMask));
-  __ j(zero, &slow);
+  GenerateKeyStringCheck(masm, rax, rcx, rbx, &index_string, &slow);
 
   // If the receiver is a fast-case object, check the keyed lookup
   // cache. Otherwise probe the dictionary leaving result in rcx.
   __ movq(rbx, FieldOperand(rdx, JSObject::kPropertiesOffset));
   __ CompareRoot(FieldOperand(rbx, HeapObject::kMapOffset),
                  Heap::kHashTableMapRootIndex);
   __ j(equal, &probe_dictionary);
 
   // Load the map of the receiver, compute the keyed lookup cache hash
   // based on 32 bits of the map pointer and the string hash.
@@ skipping 39 matching lines @@
   __ bind(&probe_dictionary);
   // rdx: receiver
   // rax: key
   GenerateDictionaryLoad(masm,
                          &slow,
                          rbx,
                          rdx,
                          rcx,
                          rax,
                          rdi,
+                         rax,
                          DICTIONARY_CHECK_DONE);
-  __ movq(rax, rdx);
   __ IncrementCounter(&Counters::keyed_load_generic_symbol, 1);
   __ ret(0);
-  // If the hash field contains an array index pick it out. The assert checks
-  // that the constants for the maximum number of digits for an array index
-  // cached in the hash field and the number of bits reserved for it does not
-  // conflict.
-  ASSERT(TenToThe(String::kMaxCachedArrayIndexLength) <
-         (1 << String::kArrayIndexValueBits));
+
   __ bind(&index_string);
-  // We want the smi-tagged index in rax. Even if we subsequently go to
-  // the slow case, converting the key to a smi is always valid.
-  // rdx: receiver
-  // rax: key (a string)
-  // rbx: key's hash field, including its array index value.
-  __ and_(rbx, Immediate(String::kArrayIndexValueMask));
-  __ shr(rbx, Immediate(String::kHashShift));
-  // Here we actually clobber the key (rax) which will be used if calling into
-  // runtime later. However as the new key is the numeric value of a string key
-  // there is no difference in using either key.
-  __ Integer32ToSmi(rax, rbx);
-  // Now jump to the place where smi keys are handled.
+  GenerateIndexFromHash(masm, rax, rbx);
   __ jmp(&index_smi);
 }
 
 
 void KeyedLoadIC::GenerateString(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- rax    : key
   //  -- rdx    : receiver
   //  -- rsp[0] : return address
   // -----------------------------------
@@ skipping 557 matching lines @@
     }
     __ ret(0);
   }
 
   // Slow case: call runtime.
   __ bind(&slow);
   GenerateRuntimeSetProperty(masm);
 }
 
 
-void CallIC::GenerateMiss(MacroAssembler* masm, int argc) {
+// Defined in ic.cc.
+Object* CallIC_Miss(Arguments args);
+
+
+static void GenerateCallMiss(MacroAssembler* masm, int argc, IC::UtilityId id) {
   // ----------- S t a t e -------------
   // rcx                      : function name
   // rsp[0]                   : return address
   // rsp[8]                   : argument argc
   // rsp[16]                  : argument argc - 1
   // ...
   // rsp[argc * 8]            : argument 1
   // rsp[(argc + 1) * 8]      : argument 0 = receiver
   // -----------------------------------
   // Get the receiver of the function from the stack; 1 ~ return address.
   __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize));
 
   // Enter an internal frame.
   __ EnterInternalFrame();
 
   // Push the receiver and the name of the function.
   __ push(rdx);
   __ push(rcx);
 
   // Call the entry.
   CEntryStub stub(1);
   __ movq(rax, Immediate(2));
-  __ movq(rbx, ExternalReference(IC_Utility(kCallIC_Miss)));
+  __ movq(rbx, ExternalReference(IC_Utility(id)));
   __ CallStub(&stub);
 
   // Move result to rdi and exit the internal frame.
   __ movq(rdi, rax);
   __ LeaveInternalFrame();
 
   // Check if the receiver is a global object of some sort.
   Label invoke, global;
   __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize));  // receiver
   __ JumpIfSmi(rdx, &invoke);
   __ CmpObjectType(rdx, JS_GLOBAL_OBJECT_TYPE, rcx);
   __ j(equal, &global);
   __ CmpInstanceType(rcx, JS_BUILTINS_OBJECT_TYPE);
   __ j(not_equal, &invoke);
 
   // Patch the receiver on the stack.
   __ bind(&global);
   __ movq(rdx, FieldOperand(rdx, GlobalObject::kGlobalReceiverOffset));
   __ movq(Operand(rsp, (argc + 1) * kPointerSize), rdx);
 
   // Invoke the function.
   ParameterCount actual(argc);
   __ bind(&invoke);
   __ InvokeFunction(rdi, actual, JUMP_FUNCTION);
 }
 
 
-// Defined in ic.cc.
-Object* CallIC_Miss(Arguments args);
-
-void CallIC::GenerateMegamorphic(MacroAssembler* masm, int argc) {
+// The generated code does not accept smi keys.
+// The generated code falls through if both probes miss.
+static void GenerateMonomorphicCacheProbe(MacroAssembler* masm,
+                                          int argc,
+                                          Code::Kind kind) {
   // ----------- S t a t e -------------
   // rcx                      : function name
-  // rsp[0]                   : return address
-  // rsp[8]                   : argument argc
-  // rsp[16]                  : argument argc - 1
-  // ...
-  // rsp[argc * 8]            : argument 1
-  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // rdx                      : receiver
   // -----------------------------------
   Label number, non_number, non_string, boolean, probe, miss;
 
-  // Get the receiver of the function from the stack; 1 ~ return address.
-  __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize));
-
   // Probe the stub cache.
   Code::Flags flags =
-      Code::ComputeFlags(Code::CALL_IC, NOT_IN_LOOP, MONOMORPHIC, NORMAL, argc);
+      Code::ComputeFlags(kind, NOT_IN_LOOP, MONOMORPHIC, NORMAL, argc);
   StubCache::GenerateProbe(masm, flags, rdx, rcx, rbx, rax);
 
   // If the stub cache probing failed, the receiver might be a value.
   // For value objects, we use the map of the prototype objects for
   // the corresponding JSValue for the cache and that is what we need
   // to probe.
   //
   // Check for number.
   __ JumpIfSmi(rdx, &number);
   __ CmpObjectType(rdx, HEAP_NUMBER_TYPE, rbx);
@@ skipping 18 matching lines @@
   __ CompareRoot(rdx, Heap::kFalseValueRootIndex);
   __ j(not_equal, &miss);
   __ bind(&boolean);
   StubCompiler::GenerateLoadGlobalFunctionPrototype(
       masm, Context::BOOLEAN_FUNCTION_INDEX, rdx);
 
   // Probe the stub cache for the value object.
   __ bind(&probe);
   StubCache::GenerateProbe(masm, flags, rdx, rcx, rbx, no_reg);
 
-  // Cache miss: Jump to runtime.
   __ bind(&miss);
-  GenerateMiss(masm, argc);
 }
 
 
 static void GenerateNormalHelper(MacroAssembler* masm,
                                  int argc,
                                  bool is_global_object,
                                  Label* miss) {
   // ----------- S t a t e -------------
   // rcx                    : function name
   // rdx                    : receiver
   // rsp[0]                 : return address
   // rsp[8]                 : argument argc
   // rsp[16]                : argument argc - 1
   // ...
   // rsp[argc * 8]          : argument 1
   // rsp[(argc + 1) * 8]    : argument 0 = receiver
   // -----------------------------------
   // Search dictionary - put result in register rdx.
-  GenerateDictionaryLoad(masm, miss, rax, rdx, rbx, rcx, rdi, CHECK_DICTIONARY);
+  GenerateDictionaryLoad(
+     masm, miss, rax, rdx, rbx, rcx, rdi, rdi, CHECK_DICTIONARY);
 
-  // Move the result to register rdi and check that it isn't a smi.
-  __ movq(rdi, rdx);
-  __ JumpIfSmi(rdx, miss);
-
+  __ JumpIfSmi(rdi, miss);
   // Check that the value is a JavaScript function.
-  __ CmpObjectType(rdx, JS_FUNCTION_TYPE, rdx);
+  __ CmpObjectType(rdi, JS_FUNCTION_TYPE, rdx);
   __ j(not_equal, miss);
 
   // Patch the receiver with the global proxy if necessary.
   if (is_global_object) {
-    __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize));
     __ movq(rdx, FieldOperand(rdx, GlobalObject::kGlobalReceiverOffset));
     __ movq(Operand(rsp, (argc + 1) * kPointerSize), rdx);
   }
 
   // Invoke the function.
   ParameterCount actual(argc);
   __ InvokeFunction(rdi, actual, JUMP_FUNCTION);
 }
 
 
-void CallIC::GenerateNormal(MacroAssembler* masm, int argc) {
+// The generated code falls through if the call should be handled by runtime.
+static void GenerateCallNormal(MacroAssembler* masm, int argc) {
   // ----------- S t a t e -------------
   // rcx                    : function name
   // rsp[0]                 : return address
   // rsp[8]                 : argument argc
   // rsp[16]                : argument argc - 1
   // ...
   // rsp[argc * 8]          : argument 1
   // rsp[(argc + 1) * 8]    : argument 0 = receiver
   // -----------------------------------
   Label miss, global_object, non_global_object;
@@ skipping 40 matching lines @@
   __ testb(rbx, Immediate(1 << Map::kIsAccessCheckNeeded));
   __ j(not_equal, &miss);
   __ bind(&invoke);
   GenerateNormalHelper(masm, argc, false, &miss);
 
   // Global object proxy access: Check access rights.
   __ bind(&global_proxy);
   __ CheckAccessGlobalProxy(rdx, rax, &miss);
   __ jmp(&invoke);
 
-  // Cache miss: Jump to runtime.
   __ bind(&miss);
+}
+
+
+void CallIC::GenerateMiss(MacroAssembler* masm, int argc) {
+  // ----------- S t a t e -------------
+  // rcx                      : function name
+  // rsp[0]                   : return address
+  // rsp[8]                   : argument argc
+  // rsp[16]                  : argument argc - 1
+  // ...
+  // rsp[argc * 8]            : argument 1
+  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // -----------------------------------
+  GenerateCallMiss(masm, argc, IC::kCallIC_Miss);
+}
+
+
+void CallIC::GenerateMegamorphic(MacroAssembler* masm, int argc) {
+  // ----------- S t a t e -------------
+  // rcx                      : function name
+  // rsp[0]                   : return address
+  // rsp[8]                   : argument argc
+  // rsp[16]                  : argument argc - 1
+  // ...
+  // rsp[argc * 8]            : argument 1
+  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // -----------------------------------
+
+  // Get the receiver of the function from the stack; 1 ~ return address.
+  __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize));
+  GenerateMonomorphicCacheProbe(masm, argc, Code::CALL_IC);
+  GenerateMiss(masm, argc);
+}
+
+
+void CallIC::GenerateNormal(MacroAssembler* masm, int argc) {
+  // ----------- S t a t e -------------
+  // rcx                      : function name
+  // rsp[0]                   : return address
+  // rsp[8]                   : argument argc
+  // rsp[16]                  : argument argc - 1
+  // ...
+  // rsp[argc * 8]            : argument 1
+  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // -----------------------------------
+
+  GenerateCallNormal(masm, argc);
   GenerateMiss(masm, argc);
 }
 
 
 void KeyedCallIC::GenerateMiss(MacroAssembler* masm, int argc) {
-  UNREACHABLE();
+  // ----------- S t a t e -------------
+  // rcx                      : function name
+  // rsp[0]                   : return address
+  // rsp[8]                   : argument argc
+  // rsp[16]                  : argument argc - 1
+  // ...
+  // rsp[argc * 8]            : argument 1
+  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // -----------------------------------
+
+  GenerateCallMiss(masm, argc, IC::kKeyedCallIC_Miss);
 }
 
 
 void KeyedCallIC::GenerateMegamorphic(MacroAssembler* masm, int argc) {
-  UNREACHABLE();
+  // ----------- S t a t e -------------
+  // rcx                      : function name
+  // rsp[0]                   : return address
+  // rsp[8]                   : argument argc
+  // rsp[16]                  : argument argc - 1
+  // ...
+  // rsp[argc * 8]            : argument 1
+  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // -----------------------------------
+
+  // Get the receiver of the function from the stack; 1 ~ return address.
+  __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize));
+
+  Label do_call, slow_call, slow_load, slow_reload_receiver;
+  Label check_number_dictionary, check_string, lookup_monomorphic_cache;
+  Label index_smi, index_string;
+
+  // Check that the key is a smi.
+  __ JumpIfNotSmi(rcx, &check_string);
+
+  __ bind(&index_smi);
+  // Now the key is known to be a smi. This place is also jumped to from below
+  // where a numeric string is converted to a smi.
+
+  GenerateKeyedLoadReceiverCheck(masm, rdx, rax, &slow_call);
+
+  GenerateFastArrayLoad(
+      masm, rdx, rcx, rax, rbx, rdi, &check_number_dictionary, &slow_load);
+  __ IncrementCounter(&Counters::keyed_call_generic_smi_fast, 1);
+
+  __ bind(&do_call);
+  // receiver in rdx is not used after this point.
+  // rcx: key
+  // rdi: function
+
+  // Check that the value in edi is a JavaScript function.
+  __ JumpIfSmi(rdi, &slow_call);
+  __ CmpObjectType(rdi, JS_FUNCTION_TYPE, rax);
+  __ j(not_equal, &slow_call);
+  // Invoke the function.
+  ParameterCount actual(argc);
+  __ InvokeFunction(rdi, actual, JUMP_FUNCTION);
+
+  __ bind(&check_number_dictionary);
+  // eax: elements
+  // ecx: smi key
+  // Check whether the elements is a number dictionary.
+  __ CompareRoot(FieldOperand(rax, HeapObject::kMapOffset),
+                 Heap::kHashTableMapRootIndex);
+  __ SmiToInteger32(rbx, rcx);
+  // ebx: untagged index
+  GenerateNumberDictionaryLoad(masm, &slow_load, rax, rcx, rbx, r9, rdi, rdi);
+  __ IncrementCounter(&Counters::keyed_call_generic_smi_dict, 1);
+  __ jmp(&do_call);
+
+  __ bind(&slow_load);
+  // This branch is taken when calling KeyedCallIC_Miss is neither required
+  // nor beneficial.
+  __ IncrementCounter(&Counters::keyed_call_generic_slow_load, 1);
+  __ EnterInternalFrame();
+  __ push(rcx);  // save the key
+  __ push(rdx);  // pass the receiver
+  __ push(rcx);  // pass the key
+  __ CallRuntime(Runtime::kKeyedGetProperty, 2);
+  __ pop(rcx);  // restore the key
+  __ LeaveInternalFrame();
+  __ movq(rdi, rax);
+  __ jmp(&do_call);
+
+  __ bind(&check_string);
+  GenerateKeyStringCheck(masm, rcx, rax, rbx, &index_string, &slow_call);
+
+  // The key is known to be a symbol.
+  // If the receiver is a regular JS object with slow properties then do
+  // a quick inline probe of the receiver's dictionary.
+  // Otherwise do the monomorphic cache probe.
+  GenerateKeyedLoadReceiverCheck(masm, rdx, rax, &lookup_monomorphic_cache);
+
+  __ movq(rbx, FieldOperand(rdx, JSObject::kPropertiesOffset));
+  __ CompareRoot(FieldOperand(rbx, HeapObject::kMapOffset),
+                 Heap::kHashTableMapRootIndex);
+  __ j(not_equal, &lookup_monomorphic_cache);
+
+  GenerateDictionaryLoad(
+      masm, &slow_load, rbx, rdx, rax, rcx, rdi, rdi, DICTIONARY_CHECK_DONE);
+  __ IncrementCounter(&Counters::keyed_call_generic_lookup_dict, 1);
+  __ jmp(&do_call);
+
+  __ bind(&lookup_monomorphic_cache);
+  __ IncrementCounter(&Counters::keyed_call_generic_lookup_cache, 1);
+  GenerateMonomorphicCacheProbe(masm, argc, Code::KEYED_CALL_IC);
+  // Fall through on miss.
+
+  __ bind(&slow_call);
+  // This branch is taken if:
+  // - the receiver requires boxing or access check,
+  // - the key is neither smi nor symbol,
+  // - the value loaded is not a function,
+  // - there is hope that the runtime will create a monomorphic call stub
+  //   that will get fetched next time.
+  __ IncrementCounter(&Counters::keyed_call_generic_slow, 1);
+  GenerateMiss(masm, argc);
+
+  __ bind(&index_string);
+  GenerateIndexFromHash(masm, rcx, rbx);
+  // Now jump to the place where smi keys are handled.
+  __ jmp(&index_smi);
 }
 
 
 void KeyedCallIC::GenerateNormal(MacroAssembler* masm, int argc) {
-  UNREACHABLE();
+  // ----------- S t a t e -------------
+  // rcx                      : function name
+  // rsp[0]                   : return address
+  // rsp[8]                   : argument argc
+  // rsp[16]                  : argument argc - 1
+  // ...
+  // rsp[argc * 8]            : argument 1
+  // rsp[(argc + 1) * 8]      : argument 0 = receiver
+  // -----------------------------------
+
+  GenerateCallNormal(masm, argc);
+  GenerateMiss(masm, argc);
 }
 
 
 // The offset from the inlined patch site to the start of the
 // inlined load instruction.
 const int LoadIC::kOffsetToLoadInstruction = 20;
 
 
 void LoadIC::ClearInlinedVersion(Address address) {
   // Reset the map check of the inlined inobject property load (if
@@ skipping 90 matching lines @@
   __ j(equal, &global);
 
   // Check for non-global object that requires access check.
   __ testl(FieldOperand(rbx, Map::kBitFieldOffset),
            Immediate(1 << Map::kIsAccessCheckNeeded));
   __ j(not_zero, &miss);
 
   // Search the dictionary placing the result in rax.
   __ bind(&probe);
   GenerateDictionaryLoad(masm, &miss, rdx, rax, rbx,
-                         rcx, rdi, CHECK_DICTIONARY);
+                         rcx, rdi, rax, CHECK_DICTIONARY);
   __ ret(0);
 
   // Global object access: Check access rights.
   __ bind(&global);
   __ CheckAccessGlobalProxy(rax, rdx, &miss);
   __ jmp(&probe);
 
   // Cache miss: Jump to runtime.
   __ bind(&miss);
   GenerateMiss(masm);
@@ skipping 130 matching lines @@
   GenerateMiss(masm);
 }
 
 
 #undef __
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_TARGET_ARCH_X64