Merge bleeding edge into the GC branch up to 7948. The asserts

src/ia32/ic-ia32.cc

Index: src/ia32/ic-ia32.cc
===================================================================
--- src/ia32/ic-ia32.cc	(revision 7948)
+++ src/ia32/ic-ia32.cc	(working copy)
@@ -50,11 +50,11 @@
   // Register usage:
   //   type: holds the receiver instance type on entry.
   __ cmp(type, JS_GLOBAL_OBJECT_TYPE);
-  __ j(equal, global_object, not_taken);
+  __ j(equal, global_object);
   __ cmp(type, JS_BUILTINS_OBJECT_TYPE);
-  __ j(equal, global_object, not_taken);
+  __ j(equal, global_object);
   __ cmp(type, JS_GLOBAL_PROXY_TYPE);
-  __ j(equal, global_object, not_taken);
+  __ j(equal, global_object);
 }
 
 
@@ -73,13 +73,13 @@
 
   // Check that the receiver isn't a smi.
   __ test(receiver, Immediate(kSmiTagMask));
-  __ j(zero, miss, not_taken);
+  __ j(zero, miss);
 
   // Check that the receiver is a valid JS object.
   __ mov(r1, FieldOperand(receiver, HeapObject::kMapOffset));
   __ movzx_b(r0, FieldOperand(r1, Map::kInstanceTypeOffset));
   __ cmp(r0, FIRST_JS_OBJECT_TYPE);
-  __ j(below, miss, not_taken);
+  __ j(below, miss);
 
   // If this assert fails, we have to check upper bound too.
   ASSERT(LAST_TYPE == JS_FUNCTION_TYPE);
@@ -90,68 +90,13 @@
   __ test_b(FieldOperand(r1, Map::kBitFieldOffset),
             (1 << Map::kIsAccessCheckNeeded) |
             (1 << Map::kHasNamedInterceptor));
-  __ j(not_zero, miss, not_taken);
+  __ j(not_zero, miss);
 
   __ mov(r0, FieldOperand(receiver, JSObject::kPropertiesOffset));
-  __ CheckMap(r0, FACTORY->hash_table_map(), miss, true);
+  __ CheckMap(r0, FACTORY->hash_table_map(), miss, DONT_DO_SMI_CHECK);
 }
 
 
-// Probe the string dictionary in the |elements| register. Jump to the
-// |done| label if a property with the given name is found leaving the
-// index into the dictionary in |r0|. Jump to the |miss| label
-// otherwise.
-static void GenerateStringDictionaryProbes(MacroAssembler* masm,
-                                           Label* miss,
-                                           Label* done,
-                                           Register elements,
-                                           Register name,
-                                           Register r0,
-                                           Register r1) {
-  // Assert that name contains a string.
-  if (FLAG_debug_code) __ AbortIfNotString(name);
-
-  // Compute the capacity mask.
-  const int kCapacityOffset =
-      StringDictionary::kHeaderSize +
-      StringDictionary::kCapacityIndex * kPointerSize;
-  __ mov(r1, FieldOperand(elements, kCapacityOffset));
-  __ shr(r1, kSmiTagSize);  // convert smi to int
-  __ dec(r1);
-
-  // Generate an unrolled loop that performs a few probes before
-  // giving up. Measurements done on Gmail indicate that 2 probes
-  // cover ~93% of loads from dictionaries.
-  static const int kProbes = 4;
-  const int kElementsStartOffset =
-      StringDictionary::kHeaderSize +
-      StringDictionary::kElementsStartIndex * kPointerSize;
-  for (int i = 0; i < kProbes; i++) {
-    // Compute the masked index: (hash + i + i * i) & mask.
-    __ mov(r0, FieldOperand(name, String::kHashFieldOffset));
-    __ shr(r0, String::kHashShift);
-    if (i > 0) {
-      __ add(Operand(r0), Immediate(StringDictionary::GetProbeOffset(i)));
-    }
-    __ and_(r0, Operand(r1));
-
-    // Scale the index by multiplying by the entry size.
-    ASSERT(StringDictionary::kEntrySize == 3);
-    __ lea(r0, Operand(r0, r0, times_2, 0));  // r0 = r0 * 3
-
-    // Check if the key is identical to the name.
-    __ cmp(name, Operand(elements, r0, times_4,
-                         kElementsStartOffset - kHeapObjectTag));
-    if (i != kProbes - 1) {
-      __ j(equal, done, taken);
-    } else {
-      __ j(not_equal, miss, not_taken);
-    }
-  }
-}
-
-
-
 // Helper function used to load a property from a dictionary backing
 // storage. This function may fail to load a property even though it is
 // in the dictionary, so code at miss_label must always call a backup
@@ -183,13 +128,13 @@
   Label done;
 
   // Probe the dictionary.
-  GenerateStringDictionaryProbes(masm,
-                                 miss_label,
-                                 &done,
-                                 elements,
-                                 name,
-                                 r0,
-                                 r1);
+  StringDictionaryLookupStub::GeneratePositiveLookup(masm,
+                                                     miss_label,
+                                                     &done,
+                                                     elements,
+                                                     name,
+                                                     r0,
+                                                     r1);
 
   // If probing finds an entry in the dictionary, r0 contains the
   // index into the dictionary. Check that the value is a normal
@@ -201,7 +146,7 @@
   const int kDetailsOffset = kElementsStartOffset + 2 * kPointerSize;
   __ test(Operand(elements, r0, times_4, kDetailsOffset - kHeapObjectTag),
           Immediate(PropertyDetails::TypeField::mask() << kSmiTagSize));
-  __ j(not_zero, miss_label, not_taken);
+  __ j(not_zero, miss_label);
 
   // Get the value at the masked, scaled index.
   const int kValueOffset = kElementsStartOffset + kPointerSize;
@@ -238,13 +183,13 @@
 
 
   // Probe the dictionary.
-  GenerateStringDictionaryProbes(masm,
-                                 miss_label,
-                                 &done,
-                                 elements,
-                                 name,
-                                 r0,
-                                 r1);
+  StringDictionaryLookupStub::GeneratePositiveLookup(masm,
+                                                     miss_label,
+                                                     &done,
+                                                     elements,
+                                                     name,
+                                                     r0,
+                                                     r1);
 
   // If probing finds an entry in the dictionary, r0 contains the
   // index into the dictionary. Check that the value is a normal
@@ -259,7 +204,7 @@
          PropertyDetails::AttributesField::encode(READ_ONLY)) << kSmiTagSize;
   __ test(Operand(elements, r0, times_4, kDetailsOffset - kHeapObjectTag),
           Immediate(kTypeAndReadOnlyMask));
-  __ j(not_zero, miss_label, not_taken);
+  __ j(not_zero, miss_label);
 
   // Store the value at the masked, scaled index.
   const int kValueOffset = kElementsStartOffset + kPointerSize;
@@ -349,9 +294,9 @@
                              times_pointer_size,
                              NumberDictionary::kElementsStartOffset));
     if (i != (kProbes - 1)) {
-      __ j(equal, &done, taken);
+      __ j(equal, &done);
     } else {
-      __ j(not_equal, miss, not_taken);
+      __ j(not_equal, miss);
     }
   }
 
@@ -429,7 +374,7 @@
 
   // Check that the object isn't a smi.
   __ test(receiver, Immediate(kSmiTagMask));
-  __ j(zero, slow, not_taken);
+  __ j(zero, slow);
 
   // Get the map of the receiver.
   __ mov(map, FieldOperand(receiver, HeapObject::kMapOffset));
@@ -437,7 +382,7 @@
   // Check bit field.
   __ test_b(FieldOperand(map, Map::kBitFieldOffset),
             (1 << Map::kIsAccessCheckNeeded) | (1 << interceptor_bit));
-  __ j(not_zero, slow, not_taken);
+  __ j(not_zero, slow);
   // Check that the object is some kind of JS object EXCEPT JS Value type.
   // In the case that the object is a value-wrapper object,
   // we enter the runtime system to make sure that indexing
@@ -445,7 +390,7 @@
   ASSERT(JS_OBJECT_TYPE > JS_VALUE_TYPE);
 
   __ CmpInstanceType(map, JS_OBJECT_TYPE);
-  __ j(below, slow, not_taken);
+  __ j(below, slow);
 }
 
 
@@ -469,7 +414,10 @@
   __ mov(scratch, FieldOperand(receiver, JSObject::kElementsOffset));
   if (not_fast_array != NULL) {
     // Check that the object is in fast mode and writable.
-    __ CheckMap(scratch, FACTORY->fixed_array_map(), not_fast_array, true);
+    __ CheckMap(scratch,
+                FACTORY->fixed_array_map(),
+                not_fast_array,
+                DONT_DO_SMI_CHECK);
   } else {
     __ AssertFastElements(scratch);
   }
@@ -508,12 +456,12 @@
   // Is the string an array index, with cached numeric value?
   __ mov(hash, FieldOperand(key, String::kHashFieldOffset));
   __ test(hash, Immediate(String::kContainsCachedArrayIndexMask));
-  __ j(zero, index_string, not_taken);
+  __ j(zero, index_string);
 
   // Is the string a symbol?
   ASSERT(kSymbolTag != 0);
   __ test_b(FieldOperand(map, Map::kInstanceTypeOffset), kIsSymbolMask);
-  __ j(zero, not_symbol, not_taken);
+  __ j(zero, not_symbol);
 }
 
 
@@ -528,7 +476,7 @@
 
   // Check that the key is a smi.
   __ test(eax, Immediate(kSmiTagMask));
-  __ j(not_zero, &check_string, not_taken);
+  __ j(not_zero, &check_string);
   __ bind(&index_smi);
   // Now the key is known to be a smi. This place is also jumped to from
   // where a numeric string is converted to a smi.
@@ -540,7 +488,7 @@
   // now in ecx.
   __ test_b(FieldOperand(ecx, Map::kBitField2Offset),
             1 << Map::kHasFastElements);
-  __ j(zero, &check_number_dictionary, not_taken);
+  __ j(zero, &check_number_dictionary);
 
   GenerateFastArrayLoad(masm,
                         edx,
@@ -564,7 +512,10 @@
   // ebx: untagged index
   // eax: key
   // ecx: elements
-  __ CheckMap(ecx, isolate->factory()->hash_table_map(), &slow, true);
+  __ CheckMap(ecx,
+              isolate->factory()->hash_table_map(),
+              &slow,
+              DONT_DO_SMI_CHECK);
   Label slow_pop_receiver;
   // Push receiver on the stack to free up a register for the dictionary
   // probing.
@@ -704,7 +655,7 @@
   char_at_generator.GenerateSlow(masm, call_helper);
 
   __ bind(&miss);
-  GenerateMiss(masm);
+  GenerateMiss(masm, false);
 }
 
 
@@ -718,11 +669,11 @@
 
   // Check that the receiver isn't a smi.
   __ test(edx, Immediate(kSmiTagMask));
-  __ j(zero, &slow, not_taken);
+  __ j(zero, &slow);
 
   // Check that the key is an array index, that is Uint32.
   __ test(eax, Immediate(kSmiTagMask | kSmiSignMask));
-  __ j(not_zero, &slow, not_taken);
+  __ j(not_zero, &slow);
 
   // Get the map of the receiver.
   __ mov(ecx, FieldOperand(edx, HeapObject::kMapOffset));
@@ -732,7 +683,7 @@
   __ movzx_b(ecx, FieldOperand(ecx, Map::kBitFieldOffset));
   __ and_(Operand(ecx), Immediate(kSlowCaseBitFieldMask));
   __ cmp(Operand(ecx), Immediate(1 << Map::kHasIndexedInterceptor));
-  __ j(not_zero, &slow, not_taken);
+  __ j(not_zero, &slow);
 
   // Everything is fine, call runtime.
   __ pop(ecx);
@@ -747,7 +698,7 @@
   __ TailCallExternalReference(ref, 2, 1);
 
   __ bind(&slow);
-  GenerateMiss(masm);
+  GenerateMiss(masm, false);
 }
 
 
@@ -763,22 +714,22 @@
 
   // Check that the object isn't a smi.
   __ test(edx, Immediate(kSmiTagMask));
-  __ j(zero, &slow, not_taken);
+  __ j(zero, &slow);
   // Get the map from the receiver.
   __ mov(edi, FieldOperand(edx, HeapObject::kMapOffset));
   // Check that the receiver does not require access checks.  We need
   // to do this because this generic stub does not perform map checks.
   __ test_b(FieldOperand(edi, Map::kBitFieldOffset),
             1 << Map::kIsAccessCheckNeeded);
-  __ j(not_zero, &slow, not_taken);
+  __ j(not_zero, &slow);
   // Check that the key is a smi.
   __ test(ecx, Immediate(kSmiTagMask));
-  __ j(not_zero, &slow, not_taken);
+  __ j(not_zero, &slow);
   __ CmpInstanceType(edi, JS_ARRAY_TYPE);
   __ j(equal, &array);
   // Check that the object is some kind of JS object.
   __ CmpInstanceType(edi, FIRST_JS_OBJECT_TYPE);
-  __ j(below, &slow, not_taken);
+  __ j(below, &slow);
 
   // Object case: Check key against length in the elements array.
   // eax: value
@@ -786,9 +737,9 @@
   // ecx: key (a smi)
   __ mov(edi, FieldOperand(edx, JSObject::kElementsOffset));
   // Check that the object is in fast mode and writable.
-  __ CheckMap(edi, FACTORY->fixed_array_map(), &slow, true);
+  __ CheckMap(edi, FACTORY->fixed_array_map(), &slow, DONT_DO_SMI_CHECK);
   __ cmp(ecx, FieldOperand(edi, FixedArray::kLengthOffset));
-  __ j(below, &fast, taken);
+  __ j(below, &fast);
 
   // Slow case: call runtime.
   __ bind(&slow);
@@ -803,9 +754,10 @@
   // ecx: key, a smi.
   // edi: receiver->elements, a FixedArray
   // flags: compare (ecx, edx.length())
-  __ j(not_equal, &slow, not_taken);  // do not leave holes in the array
+  // do not leave holes in the array:
+  __ j(not_equal, &slow);
   __ cmp(ecx, FieldOperand(edi, FixedArray::kLengthOffset));
-  __ j(above_equal, &slow, not_taken);
+  __ j(above_equal, &slow);
   // Add 1 to receiver->length, and go to fast array write.
   __ add(FieldOperand(edx, JSArray::kLengthOffset),
          Immediate(Smi::FromInt(1)));
@@ -819,12 +771,12 @@
   // edx: receiver, a JSArray
   // ecx: key, a smi.
   __ mov(edi, FieldOperand(edx, JSObject::kElementsOffset));
-  __ CheckMap(edi, FACTORY->fixed_array_map(), &slow, true);
+  __ CheckMap(edi, FACTORY->fixed_array_map(), &slow, DONT_DO_SMI_CHECK);
 
   // Check the key against the length in the array, compute the
   // address to store into and fall through to fast case.
   __ cmp(ecx, FieldOperand(edx, JSArray::kLengthOffset));  // Compare smis.
-  __ j(above_equal, &extra, not_taken);
+  __ j(above_equal, &extra);
 
   // Fast case: Do the store.
   __ bind(&fast);
@@ -869,9 +821,9 @@
   //
   // Check for number.
   __ test(edx, Immediate(kSmiTagMask));
-  __ j(zero, &number, not_taken);
+  __ j(zero, &number);
   __ CmpObjectType(edx, HEAP_NUMBER_TYPE, ebx);
-  __ j(not_equal, &non_number, taken);
+  __ j(not_equal, &non_number);
   __ bind(&number);
   StubCompiler::GenerateLoadGlobalFunctionPrototype(
       masm, Context::NUMBER_FUNCTION_INDEX, edx);
@@ -880,7 +832,7 @@
   // Check for string.
   __ bind(&non_number);
   __ CmpInstanceType(ebx, FIRST_NONSTRING_TYPE);
-  __ j(above_equal, &non_string, taken);
+  __ j(above_equal, &non_string);
   StubCompiler::GenerateLoadGlobalFunctionPrototype(
       masm, Context::STRING_FUNCTION_INDEX, edx);
   __ jmp(&probe);
@@ -888,9 +840,9 @@
   // Check for boolean.
   __ bind(&non_string);
   __ cmp(edx, FACTORY->true_value());
-  __ j(equal, &boolean, not_taken);
+  __ j(equal, &boolean);
   __ cmp(edx, FACTORY->false_value());
-  __ j(not_equal, &miss, taken);
+  __ j(not_equal, &miss);
   __ bind(&boolean);
   StubCompiler::GenerateLoadGlobalFunctionPrototype(
       masm, Context::BOOLEAN_FUNCTION_INDEX, edx);
@@ -917,11 +869,11 @@
 
   // Check that the result is not a smi.
   __ test(edi, Immediate(kSmiTagMask));
-  __ j(zero, miss, not_taken);
+  __ j(zero, miss);
 
   // Check that the value is a JavaScript function, fetching its map into eax.
   __ CmpObjectType(edi, JS_FUNCTION_TYPE, eax);
-  __ j(not_equal, miss, not_taken);
+  __ j(not_equal, miss);
 
   // Invoke the function.
   ParameterCount actual(argc);
@@ -997,13 +949,13 @@
     Label invoke, global;
     __ mov(edx, Operand(esp, (argc + 1) * kPointerSize));  // receiver
     __ test(edx, Immediate(kSmiTagMask));
-    __ j(zero, &invoke, not_taken);
+    __ j(zero, &invoke, Label::kNear);
     __ mov(ebx, FieldOperand(edx, HeapObject::kMapOffset));
     __ movzx_b(ebx, FieldOperand(ebx, Map::kInstanceTypeOffset));
     __ cmp(ebx, JS_GLOBAL_OBJECT_TYPE);
-    __ j(equal, &global);
+    __ j(equal, &global, Label::kNear);
     __ cmp(ebx, JS_BUILTINS_OBJECT_TYPE);
-    __ j(not_equal, &invoke);
+    __ j(not_equal, &invoke, Label::kNear);
 
     // Patch the receiver on the stack.
     __ bind(&global);
@@ -1079,7 +1031,7 @@
 
   // Check that the key is a smi.
   __ test(ecx, Immediate(kSmiTagMask));
-  __ j(not_zero, &check_string, not_taken);
+  __ j(not_zero, &check_string);
 
   __ bind(&index_smi);
   // Now the key is known to be a smi. This place is also jumped to from
@@ -1104,7 +1056,10 @@
   // eax: elements
   // ecx: smi key
   // Check whether the elements is a number dictionary.
-  __ CheckMap(eax, isolate->factory()->hash_table_map(), &slow_load, true);
+  __ CheckMap(eax,
+              isolate->factory()->hash_table_map(),
+              &slow_load,
+              DONT_DO_SMI_CHECK);
   __ mov(ebx, ecx);
   __ SmiUntag(ebx);
   // ebx: untagged index
@@ -1145,7 +1100,7 @@
   __ CheckMap(ebx,
               isolate->factory()->hash_table_map(),
               &lookup_monomorphic_cache,
-              true);
+              DONT_DO_SMI_CHECK);
 
   GenerateDictionaryLoad(masm, &slow_load, ebx, ecx, eax, edi, edi);
   __ IncrementCounter(counters->keyed_call_generic_lookup_dict(), 1);
@@ -1268,7 +1223,7 @@
 }
 
 
-void KeyedLoadIC::GenerateMiss(MacroAssembler* masm) {
+void KeyedLoadIC::GenerateMiss(MacroAssembler* masm, bool force_generic) {
   // ----------- S t a t e -------------
   //  -- eax    : key
   //  -- edx    : receiver
@@ -1283,8 +1238,10 @@
   __ push(ebx);  // return address
 
   // Perform tail call to the entry.
-  ExternalReference ref =
-      ExternalReference(IC_Utility(kKeyedLoadIC_Miss), masm->isolate());
+  ExternalReference ref = force_generic
+      ? ExternalReference(IC_Utility(kKeyedLoadIC_MissForceGeneric),
+                          masm->isolate())
+      : ExternalReference(IC_Utility(kKeyedLoadIC_Miss), masm->isolate());
   __ TailCallExternalReference(ref, 2, 1);
 }
 
@@ -1369,22 +1326,22 @@
 
   // Check that the receiver isn't a smi.
   __ test(receiver, Immediate(kSmiTagMask));
-  __ j(zero, &miss, not_taken);
+  __ j(zero, &miss);
 
   // Check that the object is a JS array.
   __ CmpObjectType(receiver, JS_ARRAY_TYPE, scratch);
-  __ j(not_equal, &miss, not_taken);
+  __ j(not_equal, &miss);
 
   // Check that elements are FixedArray.
   // We rely on StoreIC_ArrayLength below to deal with all types of
   // fast elements (including COW).
   __ mov(scratch, FieldOperand(receiver, JSArray::kElementsOffset));
   __ CmpObjectType(scratch, FIXED_ARRAY_TYPE, scratch);
-  __ j(not_equal, &miss, not_taken);
+  __ j(not_equal, &miss);
 
   // Check that value is a smi.
   __ test(value, Immediate(kSmiTagMask));
-  __ j(not_zero, &miss, not_taken);
+  __ j(not_zero, &miss);
 
   // Prepare tail call to StoreIC_ArrayLength.
   __ pop(scratch);
@@ -1476,7 +1433,7 @@
 }
 
 
-void KeyedStoreIC::GenerateMiss(MacroAssembler* masm) {
+void KeyedStoreIC::GenerateMiss(MacroAssembler* masm, bool force_generic) {
   // ----------- S t a t e -------------
   //  -- eax    : value
   //  -- ecx    : key
@@ -1491,12 +1448,34 @@
   __ push(ebx);
 
   // Do tail-call to runtime routine.
-  ExternalReference ref =
-      ExternalReference(IC_Utility(kKeyedStoreIC_Miss), masm->isolate());
+  ExternalReference ref = force_generic
+      ? ExternalReference(IC_Utility(kKeyedStoreIC_MissForceGeneric),
+                          masm->isolate())
+      : ExternalReference(IC_Utility(kKeyedStoreIC_Miss), masm->isolate());
   __ TailCallExternalReference(ref, 3, 1);
 }
 
 
+void KeyedStoreIC::GenerateSlow(MacroAssembler* masm) {
+  // ----------- S t a t e -------------
+  //  -- eax    : value
+  //  -- ecx    : key
+  //  -- edx    : receiver
+  //  -- esp[0] : return address
+  // -----------------------------------
+
+  __ pop(ebx);
+  __ push(edx);
+  __ push(ecx);
+  __ push(eax);
+  __ push(ebx);   // return address
+
+  // Do tail-call to runtime routine.
+  ExternalReference ref(IC_Utility(kKeyedStoreIC_Slow), masm->isolate());
+  __ TailCallExternalReference(ref, 3, 1);
+}
+
+
 #undef __