Better support for 'polymorphic' JS and external arrays

src/ic.cc

Index: src/ic.cc
diff --git a/src/ic.cc b/src/ic.cc
index e2089ff155d3d32074fb9296825132872ea176d1..a843a736a4e9f99002a7021fdf2b7b9f61a100bf 100644
--- a/src/ic.cc
+++ b/src/ic.cc
@@ -1,4 +1,4 @@
-// Copyright 2006-2009 the V8 project authors. All rights reserved.
+// Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
@@ -67,7 +67,33 @@ void IC::TraceIC(const char* type,
     State new_state = StateFrom(new_target,
                                 HEAP->undefined_value(),
                                 HEAP->undefined_value());
-    PrintF("[%s (%c->%c)%s", type,
+    PrintF("[%s in ", type);
+    StackFrameIterator it;
+    while (it.frame()->fp() != this->fp()) it.Advance();
+    StackFrame* raw_frame = it.frame();
+    if (raw_frame->is_internal()) {
+      Isolate* isolate = new_target->GetIsolate();
+      Code* apply_builtin = isolate->builtins()->builtin(
+          Builtins::kFunctionApply);
+      if (raw_frame->unchecked_code() == apply_builtin) {
+        PrintF("apply from ");
+        it.Advance();
+        raw_frame = it.frame();
+      }
+    }
+    if (raw_frame->is_java_script()) {
+      JavaScriptFrame* frame = JavaScriptFrame::cast(raw_frame);
+      Code* js_code = frame->unchecked_code();
+      // Find the function on the stack and both the active code for the
+      // function and the original code.
+      JSFunction* function = JSFunction::cast(frame->function());
+      function->PrintName();
+      int code_offset = address() - js_code->instruction_start();
+      PrintF("+%d", code_offset);
+    } else {
+      PrintF("<unknown>");
+    }
+    PrintF(" (%c->%c)%s",
            TransitionMarkFromState(old_state),
            TransitionMarkFromState(new_state),
            extra_info);
@@ -274,11 +300,9 @@ void IC::Clear(Address address) {
   switch (target->kind()) {
     case Code::LOAD_IC: return LoadIC::Clear(address, target);
     case Code::KEYED_LOAD_IC:
-    case Code::KEYED_EXTERNAL_ARRAY_LOAD_IC:
       return KeyedLoadIC::Clear(address, target);
     case Code::STORE_IC: return StoreIC::Clear(address, target);
     case Code::KEYED_STORE_IC:
-    case Code::KEYED_EXTERNAL_ARRAY_STORE_IC:
       return KeyedStoreIC::Clear(address, target);
     case Code::CALL_IC: return CallIC::Clear(address, target);
     case Code::KEYED_CALL_IC:  return KeyedCallIC::Clear(address, target);
@@ -1031,9 +1055,49 @@ void LoadIC::UpdateCaches(LookupResult* lookup,
 }
 
 
+String* KeyedLoadIC::GetStubNameForCache(IC::State ic_state) {
+  if (ic_state == MONOMORPHIC) {
+    return isolate()->heap()->KeyedLoadSpecializedMonomorphic_symbol();
+  } else {
+    ASSERT(ic_state == MEGAMORPHIC);
+    return isolate()->heap()->KeyedLoadSpecializedPolymorphic_symbol();
+  }
+}
+
+
+MaybeObject* KeyedLoadIC::GetFastElementStubWithoutMapCheck(
+    bool is_js_array) {
+  return KeyedLoadFastElementStub().TryGetCode();
+}
+
+
+MaybeObject* KeyedLoadIC::GetExternalArrayStubWithoutMapCheck(
+    ExternalArrayType array_type) {
+  return KeyedLoadExternalArrayStub(array_type).TryGetCode();
+}
+
+
+MaybeObject* KeyedLoadIC::ConstructMegamorphicStub(
+    MapList* receiver_maps,
+    CodeList* targets,
+    StrictModeFlag strict_mode) {
+  Object* object;
+  KeyedLoadStubCompiler compiler;
+  MaybeObject* maybe_code = compiler.CompileLoadMegamorphic(receiver_maps,
+                                                            targets);
+  if (!maybe_code->ToObject(&object)) return maybe_code;
+  isolate()->counters()->keyed_load_polymorphic_stubs()->Increment();
+  PROFILE(isolate(), CodeCreateEvent(
+      Logger::KEYED_LOAD_MEGAMORPHIC_IC_TAG,
+      Code::cast(object), 0));
+  return object;
+}
+
+
 MaybeObject* KeyedLoadIC::Load(State state,
                                Handle<Object> object,
-                               Handle<Object> key) {
+                               Handle<Object> key,
+                               bool force_generic_stub) {
   // Check for values that can be converted into a symbol.
   // TODO(1295): Remove this code.
   HandleScope scope(isolate());
@@ -1159,34 +1223,31 @@ MaybeObject* KeyedLoadIC::Load(State state,
 
   if (use_ic) {
     Code* stub = generic_stub();
-    if (state == UNINITIALIZED) {
+    if (!force_generic_stub) {
       if (object->IsString() && key->IsNumber()) {
-        stub = string_stub();
+        if (state == UNINITIALIZED) {
+          stub = string_stub();
+        }
       } else if (object->IsJSObject()) {
-        Handle<JSObject> receiver = Handle<JSObject>::cast(object);
-        if (receiver->HasExternalArrayElements()) {
-          MaybeObject* probe =
-              isolate()->stub_cache()->ComputeKeyedLoadOrStoreExternalArray(
-                  *receiver, false, kNonStrictMode);
-          stub = probe->IsFailure() ?
-              NULL : Code::cast(probe->ToObjectUnchecked());
-        } else if (receiver->HasIndexedInterceptor()) {
+        JSObject* receiver = JSObject::cast(*object);
+        if (receiver->HasIndexedInterceptor()) {
           stub = indexed_interceptor_stub();
-        } else if (key->IsSmi() &&
-                   receiver->map()->has_fast_elements()) {
-          MaybeObject* probe =
-              isolate()->stub_cache()->ComputeKeyedLoadSpecialized(*receiver);
-          stub = probe->IsFailure() ?
-              NULL : Code::cast(probe->ToObjectUnchecked());
+        } else if (key->IsSmi()) {
+          MaybeObject* maybe_stub = ComputeStub(receiver,
+                                                false,
+                                                kNonStrictMode,
+                                                stub);
+          stub = maybe_stub->IsFailure() ?
+              NULL : Code::cast(maybe_stub->ToObjectUnchecked());
         }
       }
     }
     if (stub != NULL) set_target(stub);
+  }
 
 #ifdef DEBUG
-    TraceIC("KeyedLoadIC", key, state, target());
+  TraceIC("KeyedLoadIC", key, state, target());
 #endif  // DEBUG
-  }
 
   // Get the property.
   return Runtime::GetObjectProperty(isolate(), object, key);
@@ -1483,11 +1544,256 @@ void StoreIC::UpdateCaches(LookupResult* lookup,
 }
 
 
+static bool AddOneReceiverMapIfMissing(MapList* receiver_maps,
+                                       Map* new_receiver_map) {
+  for (int current = 0; current < receiver_maps->length(); ++current) {
+    if (receiver_maps->at(current) == new_receiver_map) {
+      return false;
+    }
+  }
+  receiver_maps->Add(new_receiver_map);
+  return true;
+}
+
+
+void KeyedIC::GetReceiverMapsForStub(Code* stub, MapList* result) {
+  ASSERT(stub->is_inline_cache_stub());
+  if (stub == string_stub()) {
+    return result->Add(isolate()->heap()->string_map());
+  } else if (stub->is_keyed_load_stub() || stub->is_keyed_store_stub()) {
+    if (stub->ic_state() == MONOMORPHIC) {
+      result->Add(Map::cast(stub->FindFirstMap()));
+    } else {
+      ASSERT(stub->ic_state() == MEGAMORPHIC);
+      AssertNoAllocation no_allocation;
+      int mask = RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT);
+      for (RelocIterator it(stub, mask); !it.done(); it.next()) {
+        RelocInfo* info = it.rinfo();
+        Object* object = info->target_object();
+        ASSERT(object->IsMap());
+        result->Add(Map::cast(object));
+      }
+    }
+  }
+}
+
+
+MaybeObject* KeyedIC::ComputeStub(JSObject* receiver,
+                                  bool is_store,
+                                  StrictModeFlag strict_mode,
+                                  Code* generic_stub) {
+  State ic_state = target()->ic_state();
+  Code* monomorphic_stub;
+  // Always compute the MONOMORPHIC stub, even if the MEGAMORPHIC stub ends up
+  // being used. This is necessary because the megamorphic stub needs to have
+  // access to more information than what is stored in the receiver map in some
+  // cases (external arrays need the array type from the MONOMORPHIC stub).

[Mads Ager (chromium), 2011/05/12 09:42:47]

I don't know how much that will happen, but we shouldn't generate the
monomorphic fast elements stub unless we need them. So we shouldn't generate
them if we are (going) megamorphic. They are not needed later in that case.

That should be simple to avoid?

[danno, 2011/06/01 13:16:08]

It's not the easy here, unfortunately. We need to make sure that the array type
is available from the monomorphic stub, otherwise we don't know which external
array shared stub to use in the megamorphic case (thus the comment). I'll add a
TODO that when we finally figure out a better way to track the external array
type with the map that we should remove this. 

+  MaybeObject* maybe_stub = ComputeMonomorphicStub(receiver,
+                                                   is_store,
+                                                   strict_mode,
+                                                   generic_stub);
+  if (!maybe_stub->To(&monomorphic_stub)) return maybe_stub;
+
+  if (ic_state == UNINITIALIZED || ic_state == PREMONOMORPHIC) {
+    return monomorphic_stub;
+  }
+  ASSERT(target() != generic_stub);
+
+  // Don't handle megamorphic property accesses for INTERCEPTORS or CALLBACKS
+  // via megamorphic stubs, since they don't have a map in their relocation info
+  // and so the stubs can't be harvested for the object needed for a map check.
+  if (target()->type() != NORMAL) {
+    return generic_stub;
+  }
+
+  // Determine the list of receiver maps that this call site has seen,
+  // adding the map that was just encountered.
+  MapList target_receiver_maps;
+  GetReceiverMapsForStub(target(), &target_receiver_maps);
+  if (!AddOneReceiverMapIfMissing(&target_receiver_maps, receiver->map())) {
+    // If the miss wasn't due to an unseen map, a MEGAMORPHIC stub
+    // won't help, use the generic stub.
+    return generic_stub;
+  }
+
+  // TODO(1385): Currently MEGAMORPHIC stubs are cached in the receiver map stub
+  // cache, but that can put receiver types together from unrelated call sites
+  // into the same stub--they always handle the union of all receiver maps seen
+  // at all call sites involving the receiver map. This is only an
+  // approximation: ideally, there would be a global cache that mapped sets of
+  // receiver maps to MEGAMORPHIC stubs. The complexity of the MEGAMORPHIC stub
+  // computation also leads to direct manipulation of the stub cache from the IC
+  // code, which the global cache solution would avoid.
+  Code::Kind kind = this->kind();
+  Code::Flags flags = Code::ComputeFlags(kind,
+                                         NOT_IN_LOOP,
+                                         MEGAMORPHIC,
+                                         strict_mode);
+  String* megamorphic_name = GetStubNameForCache(MEGAMORPHIC);
+  Object* maybe_cached_stub = receiver->map()->FindInCodeCache(megamorphic_name,
+                                                               flags);
+
+  // Create a set of all receiver maps that have been seen at the IC call site
+  // and those seen by the MEGAMORPHIC cached stub, if that's the stub that's
+  // been selected.
+  MapList receiver_maps;
+  if (!maybe_cached_stub->IsUndefined()) {
+    GetReceiverMapsForStub(Code::cast(maybe_cached_stub), &receiver_maps);
+  }
+  bool added_map = false;
+  for (int i = 0; i < target_receiver_maps.length(); ++i) {
+    if (AddOneReceiverMapIfMissing(&receiver_maps,
+                                   target_receiver_maps.at(i))) {
+      added_map = true;
+    }
+  }
+  ASSERT(receiver_maps.length() > 0);
+
+  // If the maximum number of receiver maps has been exceeded, use the Generic
+  // version of the IC.
+  if (receiver_maps.length() > KeyedIC::kMaxKeyedPolymorphism) {
+    return generic_stub;
+  }
+
+  // If no maps have been seen at the call site that aren't in the cached
+  // stub, then use it.
+  if (!added_map) {
+    ASSERT(!maybe_cached_stub->IsUndefined());
+    ASSERT(maybe_cached_stub->IsCode());
+    return Code::cast(maybe_cached_stub);
+  }
+
+  // Lookup all of the receiver maps in the cache, they should all already
+  // have MONOMORPHIC stubs.
+  CodeList handler_ics(KeyedIC::kMaxKeyedPolymorphism);
+  for (int current = 0; current < receiver_maps.length(); ++current) {
+    Map* receiver_map(receiver_maps.at(current));
+    MaybeObject* maybe_cached_stub = ComputeMonomorphicStubWithoutMapCheck(
+        receiver_map,
+        strict_mode,
+        generic_stub);
+    Code* cached_stub;
+    if (!maybe_cached_stub->To(&cached_stub)) {
+      return maybe_cached_stub;
+    }
+    handler_ics.Add(cached_stub);
+  }
+
+  Code* stub;
+  // Build the MEGAMORPHIC stub.
+  maybe_stub = ConstructMegamorphicStub(&receiver_maps,
+                                        &handler_ics,
+                                        strict_mode);
+  if (!maybe_stub->To(&stub)) return maybe_stub;
+
+  MaybeObject* maybe_update = receiver->UpdateMapCodeCache(
+      megamorphic_name,
+      stub);
+  if (maybe_update->IsFailure()) return maybe_update;
+  return stub;
+}
+
+
+MaybeObject* KeyedIC::ComputeMonomorphicStubWithoutMapCheck(
+    Map* receiver_map,
+    StrictModeFlag strict_mode,
+    Code* generic_stub) {
+  if ((receiver_map->instance_type() & kNotStringTag) == 0) {
+    ASSERT(string_stub() != NULL);
+    return string_stub();
+  } else if (receiver_map->has_external_array_elements()) {
+    // Determine the array type from the default MONOMORPHIC already generated
+    // stub. There is no other way to determine the type of the external array
+    // directly from the receiver type.
+    Code::Kind kind = this->kind();
+    Code::Flags flags = Code::ComputeMonomorphicFlags(kind,
+                                                      NORMAL,
+                                                      strict_mode);
+    String* monomorphic_name = GetStubNameForCache(MONOMORPHIC);
+    Object* maybe_default_stub = receiver_map->FindInCodeCache(monomorphic_name,
+                                                               flags);
+    if (maybe_default_stub->IsUndefined()) {
+      return generic_stub;
+    }
+    Code* default_stub = Code::cast(maybe_default_stub);
+    return GetExternalArrayStubWithoutMapCheck(
+        default_stub->external_array_type());
+  } else if (receiver_map->has_fast_elements()) {
+    bool is_js_array = receiver_map->instance_type() == JS_ARRAY_TYPE;
+    return GetFastElementStubWithoutMapCheck(is_js_array);
+  } else {
+    return generic_stub;
+  }
+}
+
+
+MaybeObject* KeyedIC::ComputeMonomorphicStub(JSObject* receiver,
+                                             bool is_store,
+                                             StrictModeFlag strict_mode,
+                                             Code* generic_stub) {
+  Code* result = NULL;
+  if (receiver->HasExternalArrayElements()) {
+    MaybeObject* maybe_stub =
+        isolate()->stub_cache()->ComputeKeyedLoadOrStoreExternalArray(
+            receiver, is_store, strict_mode);
+    if (!maybe_stub->To(&result)) return maybe_stub;
+  } else if (receiver->map()->has_fast_elements()) {
+    MaybeObject* maybe_stub =
+        isolate()->stub_cache()->ComputeKeyedLoadOrStoreFastElement(
+            receiver, is_store, strict_mode);
+    if (!maybe_stub->To(&result)) return maybe_stub;
+  } else {
+    result = generic_stub;
+  }
+  return result;
+}
+
+
+String* KeyedStoreIC::GetStubNameForCache(IC::State ic_state) {
+  if (ic_state == MONOMORPHIC) {
+    return isolate()->heap()->KeyedStoreSpecializedMonomorphic_symbol();
+  } else {
+    ASSERT(ic_state == MEGAMORPHIC);
+    return isolate()->heap()->KeyedStoreSpecializedPolymorphic_symbol();
+  }
+}
+
+
+MaybeObject* KeyedStoreIC::GetFastElementStubWithoutMapCheck(
+    bool is_js_array) {
+  return KeyedStoreFastElementStub(is_js_array).TryGetCode();
+}
+
+
+MaybeObject* KeyedStoreIC::GetExternalArrayStubWithoutMapCheck(
+    ExternalArrayType array_type) {
+  return KeyedStoreExternalArrayStub(array_type).TryGetCode();
+}
+
+
+MaybeObject* KeyedStoreIC::ConstructMegamorphicStub(
+    MapList* receiver_maps,
+    CodeList* targets,
+    StrictModeFlag strict_mode) {
+  Object* object;
+  KeyedStoreStubCompiler compiler(strict_mode);
+  MaybeObject* maybe_code = compiler.CompileStoreMegamorphic(receiver_maps,
+                                                             targets);
+  if (!maybe_code->ToObject(&object)) return maybe_code;
+  isolate()->counters()->keyed_store_polymorphic_stubs()->Increment();
+  PROFILE(isolate(), CodeCreateEvent(
+      Logger::KEYED_STORE_MEGAMORPHIC_IC_TAG,
+      Code::cast(object), 0));
+  return object;
+}
+
+
 MaybeObject* KeyedStoreIC::Store(State state,
                                  StrictModeFlag strict_mode,
                                  Handle<Object> object,
                                  Handle<Object> key,
-                                 Handle<Object> value) {
+                                 Handle<Object> value,
+                                 bool force_generic) {
   if (key->IsSymbol()) {
     Handle<String> name = Handle<String>::cast(key);
 
@@ -1529,29 +1835,27 @@ MaybeObject* KeyedStoreIC::Store(State state,
   ASSERT(!(use_ic && object->IsJSGlobalProxy()));
 
   if (use_ic) {
-    Code* stub =
-        (strict_mode == kStrictMode) ? generic_stub_strict() : generic_stub();
-    if (state == UNINITIALIZED) {
-      if (object->IsJSObject()) {
-        Handle<JSObject> receiver = Handle<JSObject>::cast(object);
-        if (receiver->HasExternalArrayElements()) {
-          MaybeObject* probe =
-              isolate()->stub_cache()->ComputeKeyedLoadOrStoreExternalArray(
-                  *receiver, true, strict_mode);
-          stub = probe->IsFailure() ?
-              NULL : Code::cast(probe->ToObjectUnchecked());
-        } else if (key->IsSmi() && receiver->map()->has_fast_elements()) {
-          MaybeObject* probe =
-              isolate()->stub_cache()->ComputeKeyedStoreSpecialized(
-                  *receiver, strict_mode);
-          stub = probe->IsFailure() ?
-              NULL : Code::cast(probe->ToObjectUnchecked());
-        }
+    Code* stub = (strict_mode == kStrictMode)
+        ? generic_stub_strict()
+        : generic_stub();
+    if (!force_generic) {
+      if (object->IsJSObject() && key->IsSmi()) {
+        JSObject* receiver = JSObject::cast(*object);
+        MaybeObject* maybe_stub = ComputeStub(receiver,
+                                              true,
+                                              strict_mode,
+                                              stub);
+        stub = maybe_stub->IsFailure() ?
+            NULL : Code::cast(maybe_stub->ToObjectUnchecked());
       }
     }
     if (stub != NULL) set_target(stub);
   }
 
+#ifdef DEBUG
+  TraceIC("KeyedStoreIC", key, state, target());
+#endif
+
   // Set the property.
   return Runtime::SetObjectProperty(
       isolate(), object , key, value, NONE, strict_mode);
@@ -1720,7 +2024,16 @@ RUNTIME_FUNCTION(MaybeObject*, KeyedLoadIC_Miss) {
   ASSERT(args.length() == 2);
   KeyedLoadIC ic(isolate);
   IC::State state = IC::StateFrom(ic.target(), args[0], args[1]);
-  return ic.Load(state, args.at<Object>(0), args.at<Object>(1));
+  return ic.Load(state, args.at<Object>(0), args.at<Object>(1), false);
+}
+
+
+RUNTIME_FUNCTION(MaybeObject*, KeyedLoadIC_MissForceGeneric) {
+  NoHandleAllocation na;
+  ASSERT(args.length() == 2);
+  KeyedLoadIC ic(isolate);
+  IC::State state = IC::StateFrom(ic.target(), args[0], args[1]);
+  return ic.Load(state, args.at<Object>(0), args.at<Object>(1), true);
 }
 
 
@@ -1804,7 +2117,42 @@ RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_Miss) {
                   static_cast<StrictModeFlag>(extra_ic_state & kStrictMode),
                   args.at<Object>(0),
                   args.at<Object>(1),
-                  args.at<Object>(2));
+                  args.at<Object>(2),
+                  false);
+}
+
+
+RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_Slow) {
+  NoHandleAllocation na;
+  ASSERT(args.length() == 3);
+  KeyedStoreIC ic(isolate);
+  Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state();
+  Handle<Object> object = args.at<Object>(0);
+  Handle<Object> key = args.at<Object>(1);
+  Handle<Object> value = args.at<Object>(2);
+  StrictModeFlag strict_mode =
+      static_cast<StrictModeFlag>(extra_ic_state & kStrictMode);
+  return Runtime::SetObjectProperty(isolate,
+                                    object,
+                                    key,
+                                    value,
+                                    NONE,
+                                    strict_mode);
+}
+
+
+RUNTIME_FUNCTION(MaybeObject*, KeyedStoreIC_MissForceGeneric) {
+  NoHandleAllocation na;
+  ASSERT(args.length() == 3);
+  KeyedStoreIC ic(isolate);
+  IC::State state = IC::StateFrom(ic.target(), args[0], args[1]);
+  Code::ExtraICState extra_ic_state = ic.target()->extra_ic_state();
+  return ic.Store(state,
+                  static_cast<StrictModeFlag>(extra_ic_state & kStrictMode),
+                  args.at<Object>(0),
+                  args.at<Object>(1),
+                  args.at<Object>(2),
+                  true);
 }