VM: Support bootstrapping core libraries from Kernel binaries instead of source.

runtime/vm/bootstrap.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/bootstrap.h"
 
 #include "include/dart_api.h"
 
 #include "vm/bootstrap_natives.h"
 #include "vm/class_finalizer.h"
 #include "vm/compiler.h"
 #include "vm/dart_api_impl.h"
+#include "vm/kernel.h"
+#include "vm/kernel_reader.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
 #include "vm/symbols.h"
 
 namespace dart {
 
-#define INIT_LIBRARY(index, name, source, patch)                               \
-  { index,                                                                     \
-    "dart:"#name, source,                                                      \
-    "dart:"#name"-patch", patch }                                              \
-
-typedef struct {
-  ObjectStore::BootstrapLibraryId index_;
-  const char* uri_;
-  const char** source_paths_;
-  const char* patch_uri_;
-  const char** patch_paths_;
-} bootstrap_lib_props;
+struct BootstrapLibProps {
+  ObjectStore::BootstrapLibraryId index;
+  const char* uri;
+  const char** source_paths;
+  const char* patch_uri;
+  const char** patch_paths;
+};
 
 
 enum {
   kPathsUriOffset = 0,
   kPathsFileOffset = 1,
   kPathsSourceOffset = 2,
   kPathsEntryLength = 3
 };
 
 
-static bootstrap_lib_props bootstrap_libraries[] = {
-  INIT_LIBRARY(ObjectStore::kCore,
-               core,
-               Bootstrap::core_source_paths_,
-               Bootstrap::core_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kAsync,
-               async,
-               Bootstrap::async_source_paths_,
-               Bootstrap::async_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kConvert,
-               convert,
-               Bootstrap::convert_source_paths_,
-               Bootstrap::convert_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kCollection,
-               collection,
-               Bootstrap::collection_source_paths_,
-               Bootstrap::collection_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kDeveloper,
-               developer,
-               Bootstrap::developer_source_paths_,
-               Bootstrap::developer_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kInternal,
-               _internal,
-               Bootstrap::_internal_source_paths_,
-               Bootstrap::_internal_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kIsolate,
-               isolate,
-               Bootstrap::isolate_source_paths_,
-               Bootstrap::isolate_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kMath,
-               math,
-               Bootstrap::math_source_paths_,
-               Bootstrap::math_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kMirrors,
-               mirrors,
-               Bootstrap::mirrors_source_paths_,
-               Bootstrap::mirrors_patch_paths_),
-  INIT_LIBRARY(ObjectStore::kProfiler,
-               profiler,
-               Bootstrap::profiler_source_paths_,
-               NULL),
-  INIT_LIBRARY(ObjectStore::kTypedData,
-               typed_data,
-               Bootstrap::typed_data_source_paths_,
-               NULL),
-  INIT_LIBRARY(ObjectStore::kVMService,
-               _vmservice,
-               Bootstrap::_vmservice_source_paths_,
-               Bootstrap::_vmservice_patch_paths_),
-  { ObjectStore::kNone, NULL, NULL, NULL, NULL }
+const char** Bootstrap::profiler_patch_paths_ = NULL;
+const char** Bootstrap::typed_data_patch_paths_ = NULL;
+
+
+#define MAKE_PROPERTIES(CamelName, hacker_name)                                \

[Florian Schneider, 2016/11/08 20:41:39]

I do like reducing boilerplate code, but I don't like mixing refactorings into
this CL.

Please factor out any changes that don't add functionality, but only refactor
the code into separate CLs. It makes this CL smaller, and clear what is needed
for functionality-wise.

[Vyacheslav Egorov (Google), 2016/11/09 14:43:16]

Moved to a separate CL: https://codereview.chromium.org/2493473002/

+  { ObjectStore::k##CamelName,                                                 \
+    "dart:" #hacker_name,                                                      \

[Florian Schneider, 2016/11/08 20:41:39]

What's a hacker_name? Why not just use "name"?

[Vyacheslav Egorov (Google), 2016/11/09 14:43:16]

I guess hacker_name is name_with_underscores. Renamed it to name.

[Kevin Millikin (Google), 2016/11/09 15:17:32]

Because the other parameter is also a name.

+    Bootstrap::hacker_name##_source_paths_,                                    \
+    "dart:" #hacker_name "-patch",                                             \
+    Bootstrap::hacker_name##_patch_paths_                                      \
+  },
+
+static BootstrapLibProps bootstrap_libraries[] = {

[Florian Schneider, 2016/11/08 20:41:39]

static const?

[Vyacheslav Egorov (Google), 2016/11/09 14:43:16]

Done.

+  FOR_EACH_BOOTSTRAP_LIBRARY(MAKE_PROPERTIES)
 };
 
+#undef MAKE_PROPERTIES
 
-static RawString* GetLibrarySource(const Library& lib,
-                                   const String& uri,
-                                   bool patch) {
-  // First check if this is a valid bootstrap library and find it's index
-  // in the 'bootstrap_libraries' table above.
-  intptr_t index;
-  const String& lib_uri = String::Handle(lib.url());
-  for (index = 0;
-       bootstrap_libraries[index].index_ != ObjectStore::kNone;
-       ++index) {
-    if (lib_uri.Equals(bootstrap_libraries[index].uri_)) {
-      break;
-    }
-  }
-  if (bootstrap_libraries[index].index_ == ObjectStore::kNone) {
-    return String::null();  // Library is not a bootstrap library.
-  }
+
+static const intptr_t kBootstrapLibraryCount = ARRAY_SIZE(bootstrap_libraries);
+
+
+static RawString* GetLibrarySourceByIndex(intptr_t index,
+                                          const String& uri,
+                                          bool patch) {
+  ASSERT(index >= 0 && index < kBootstrapLibraryCount);
 
   // Try to read the source using the path specified for the uri.
-  const char** source_paths = patch ?
-      bootstrap_libraries[index].patch_paths_ :
-      bootstrap_libraries[index].source_paths_;
+  const char** source_paths = patch ? bootstrap_libraries[index].patch_paths
+                                    : bootstrap_libraries[index].source_paths;
   if (source_paths == NULL) {
     return String::null();  // No path mapping information exists for library.
   }
   const char* source_path = NULL;
   const char* source_data = NULL;
   for (intptr_t i = 0; source_paths[i] != NULL; i += kPathsEntryLength) {
     if (uri.Equals(source_paths[i + kPathsUriOffset])) {
       source_path = source_paths[i + kPathsFileOffset];
       source_data = source_paths[i + kPathsSourceOffset];
       break;
@@ skipping 24 matching lines @@
     } else {
       return String::null();
     }
   }
   ASSERT(utf8_array != NULL);
   ASSERT(file_length >= 0);
   return String::FromUTF8(utf8_array, file_length);
 }
 
 
+static RawString* GetLibrarySource(const Library& lib,
+                                   const String& uri,
+                                   bool patch) {
+  // First check if this is a valid bootstrap library and find its index in
+  // the 'bootstrap_libraries' table above.
+  intptr_t index;
+  const String& lib_uri = String::Handle(lib.url());
+  for (index = 0; index < kBootstrapLibraryCount; ++index) {
+    if (lib_uri.Equals(bootstrap_libraries[index].uri)) {
+      break;
+    }
+  }
+  if (index == kBootstrapLibraryCount) {
+    return String::null();  // The library is not a bootstrap library.
+  }
+
+  return GetLibrarySourceByIndex(index, uri, patch);
+}
+
+
 static RawError* Compile(const Library& library, const Script& script) {
   bool update_lib_status = (script.kind() == RawScript::kScriptTag ||
                             script.kind() == RawScript::kLibraryTag);
   if (update_lib_status) {
     library.SetLoadInProgress();
   }
   const Error& error = Error::Handle(Compiler::Compile(library, script));
   if (update_lib_status) {
     if (error.IsNull()) {
       library.SetLoaded();
@@ skipping 63 matching lines @@
     return Api::NewError("Invalid import of '%s' in a bootstrap library",
                          uri_str.ToCString());
   }
   ASSERT(tag == Dart_kSourceTag);
   const Library& lib = Api::UnwrapLibraryHandle(zone, library);
   ASSERT(!lib.IsNull());
   return LoadPartSource(thread, lib, uri_str);
 }
 
 
-static RawError* LoadPatchFiles(Zone* zone,
+static RawError* LoadPatchFiles(Thread* thread,
                                 const Library& lib,
-                                const String& patch_uri,
-                                const char** patch_files) {
+                                intptr_t index) {
+  const char** patch_files = bootstrap_libraries[index].patch_paths;
+  if (patch_files == NULL) return Error::null();
+
+  Zone* zone = thread->zone();
+  String& patch_uri = String::Handle(
+      zone, Symbols::New(thread, bootstrap_libraries[index].patch_uri));
   String& patch_file_uri = String::Handle(zone);
   String& source = String::Handle(zone);
   Script& script = Script::Handle(zone);
   Error& error = Error::Handle(zone);
   const Array& strings = Array::Handle(zone, Array::New(3));
   strings.SetAt(0, patch_uri);
   strings.SetAt(1, Symbols::Slash());
   for (intptr_t j = 0; patch_files[j] != NULL; j += kPathsEntryLength) {
     patch_file_uri = String::New(patch_files[j + kPathsUriOffset]);
-    source = GetLibrarySource(lib, patch_file_uri, true);
+    source = GetLibrarySourceByIndex(index, patch_file_uri, true);
     if (source.IsNull()) {
       const String& message = String::Handle(
           String::NewFormatted("Unable to find dart patch source for %s",
                                patch_file_uri.ToCString()));
       return ApiError::New(message);
     }
     // Prepend the patch library URI to form a unique script URI for the patch.
     strings.SetAt(2, patch_file_uri);
     patch_file_uri = String::ConcatAll(strings);
     script = Script::New(patch_file_uri, source, RawScript::kPatchTag);
     error = lib.Patch(script);
     if (!error.IsNull()) {
       return error.raw();
     }
   }
   return Error::null();
 }
 
 
-RawError* Bootstrap::LoadandCompileScripts() {
-  Thread* thread = Thread::Current();
+static void Finish(Thread* thread, bool from_kernel) {
+  Bootstrap::SetupNativeResolver();
+  if (!ClassFinalizer::ProcessPendingClasses(from_kernel)) {
+    FATAL("Error in class finalization during bootstrapping.");
+  }
+
+  // Eagerly compile the _Closure class as it is the class of all closure
+  // instances. This allows us to just finalize function types without going
+  // through the hoops of trying to compile their scope class.
+  ObjectStore* object_store = thread->isolate()->object_store();
+  Class& cls = Class::Handle(thread->zone(), object_store->closure_class());
+  Compiler::CompileClass(cls);
+  // Eagerly compile Bool class, bool constants are used from within compiler.
+  cls = object_store->bool_class();
+  Compiler::CompileClass(cls);
+}
+
+
+static RawError* BootstrapFromSource(Thread* thread) {
   Isolate* isolate = thread->isolate();
   Zone* zone = thread->zone();
   String& uri = String::Handle(zone);
-  String& patch_uri = String::Handle(zone);
   String& source = String::Handle(zone);
   Script& script = Script::Handle(zone);
   Library& lib = Library::Handle(zone);
   Error& error = Error::Handle(zone);
-  Dart_LibraryTagHandler saved_tag_handler = isolate->library_tag_handler();
 
   // Set the library tag handler for the isolate to the bootstrap
   // library tag handler so that we can load all the bootstrap libraries.
+  Dart_LibraryTagHandler saved_tag_handler = isolate->library_tag_handler();
   isolate->set_library_tag_handler(BootstrapLibraryTagHandler);
 
-  HANDLESCOPE(thread);
-
-  // Create library objects for all the bootstrap libraries.
-  for (intptr_t i = 0;
-       bootstrap_libraries[i].index_ != ObjectStore::kNone;
-       ++i) {
-#ifdef PRODUCT
-    if (bootstrap_libraries[i].index_ == ObjectStore::kMirrors) {
-      continue;
-    }
-#endif  // !PRODUCT
-    uri = Symbols::New(thread, bootstrap_libraries[i].uri_);
-    lib = Library::LookupLibrary(thread, uri);
-    if (lib.IsNull()) {
-      lib = Library::NewLibraryHelper(uri, false);
-      lib.SetLoadRequested();
-      lib.Register(thread);
-    }
-    isolate->object_store()->set_bootstrap_library(
-        bootstrap_libraries[i].index_, lib);
-  }
-
   // Load, compile and patch bootstrap libraries.
-  for (intptr_t i = 0;
-       bootstrap_libraries[i].index_ != ObjectStore::kNone;
-       ++i) {
-#ifdef PRODUCT
-    if (bootstrap_libraries[i].index_ == ObjectStore::kMirrors) {
-      continue;
-    }
-#endif  // PRODUCT
-    uri = Symbols::New(thread, bootstrap_libraries[i].uri_);
-    lib = Library::LookupLibrary(thread, uri);
+  for (intptr_t i = 0; i < kBootstrapLibraryCount; ++i) {
+    ObjectStore::BootstrapLibraryId id = bootstrap_libraries[i].index;
+    uri = Symbols::New(thread, bootstrap_libraries[i].uri);
+    lib = isolate->object_store()->bootstrap_library(id);
     ASSERT(!lib.IsNull());
-    source = GetLibrarySource(lib, uri, false);
+    ASSERT(lib.raw() == Library::LookupLibrary(thread, uri));
+    source = GetLibrarySourceByIndex(i, uri, false);
     if (source.IsNull()) {
       const String& message = String::Handle(
           String::NewFormatted("Unable to find dart source for %s",
                                uri.ToCString()));
       error ^= ApiError::New(message);
       break;
     }
     script = Script::New(uri, source, RawScript::kLibraryTag);
     error = Compile(lib, script);
     if (!error.IsNull()) {
       break;
     }
     // If a patch exists, load and patch the script.
-    if (bootstrap_libraries[i].patch_paths_ != NULL) {
-      patch_uri = Symbols::New(thread, bootstrap_libraries[i].patch_uri_);
-      error = LoadPatchFiles(zone,
-                             lib,
-                             patch_uri,
-                             bootstrap_libraries[i].patch_paths_);
-      if (!error.IsNull()) {
-        break;
-      }
+    error = LoadPatchFiles(thread, lib, i);
+    if (!error.IsNull()) {
+      break;
     }
   }
+
   if (error.IsNull()) {
-    SetupNativeResolver();
-    ClassFinalizer::ProcessPendingClasses();
-
-    // Eagerly compile the _Closure class as it is the class of all closure
-    // instances. This allows us to just finalize function types
-    // without going through the hoops of trying to compile their scope class.
-    Class& cls =
-        Class::Handle(zone, isolate->object_store()->closure_class());
-    Compiler::CompileClass(cls);
-    // Eagerly compile Bool class, bool constants are used from within compiler.
-    cls = isolate->object_store()->bool_class();
-    Compiler::CompileClass(cls);
+    Finish(thread, /*from_kernel=*/false);
   }
-
   // Restore the library tag handler for the isolate.
   isolate->set_library_tag_handler(saved_tag_handler);
 
   return error.raw();
 }
 
+
+static RawError* BootstrapFromKernel(Thread* thread,
+                                     const uint8_t* buffer,
+                                     intptr_t buffer_size) {
+  Zone* zone = thread->zone();
+  kernel::Program* program =
+      ReadPrecompiledKernelFromBuffer(buffer, buffer_size);
+  if (program == NULL) {
+    const String& message =
+        String::Handle(zone, String::New("Failed to read Kernel file"));
+    return ApiError::New(message);
+  }
+
+  Isolate* isolate = thread->isolate();
+  // Mark the already-pending classes.  This mark bit will be used to avoid
+  // adding classes to the list more than once.
+  GrowableObjectArray& pending_classes = GrowableObjectArray::Handle(
+      zone, isolate->object_store()->pending_classes());
+  dart::Class& pending = dart::Class::Handle(zone);
+  for (intptr_t i = 0; i < pending_classes.Length(); ++i) {
+    pending ^= pending_classes.At(i);
+    pending.set_is_marked_for_parsing();
+  }
+
+  Library& library = Library::Handle(zone);
+  String& dart_name = String::Handle(zone);
+  String& kernel_name = String::Handle(zone);
+  kernel::KernelReader reader(NULL, -1, true);
+  for (intptr_t i = 0; i < kBootstrapLibraryCount; ++i) {
+    ObjectStore::BootstrapLibraryId id = bootstrap_libraries[i].index;
+    library = isolate->object_store()->bootstrap_library(id);
+    dart_name = library.url();
+    for (intptr_t j = 0; j < program->libraries().length(); ++j) {
+      kernel::Library* kernel_library = program->libraries()[j];
+      kernel::String* uri = kernel_library->import_uri();
+      kernel_name = Symbols::FromUTF8(thread, uri->buffer(), uri->size());
+      if (kernel_name.Equals(dart_name)) {
+        reader.ReadLibrary(kernel_library);
+        library.SetLoaded();
+        break;
+      }
+    }
+  }
+
+  Finish(thread, /*from_kernel=*/true);
+  return Error::null();
+}
+
+
+RawError* Bootstrap::DoBootstrapping(const uint8_t* kernel_buffer,
+                                     intptr_t kernel_buffer_length) {
+  Thread* thread = Thread::Current();
+  Isolate* isolate = thread->isolate();
+  Zone* zone = thread->zone();
+  String& uri = String::Handle(zone);
+  Library& lib = Library::Handle(zone);
+
+  HANDLESCOPE(thread);
+
+  // Ensure there are library objects for all the bootstrap libraries.
+  for (intptr_t i = 0; i < kBootstrapLibraryCount; ++i) {
+    ObjectStore::BootstrapLibraryId id = bootstrap_libraries[i].index;
+    uri = Symbols::New(thread, bootstrap_libraries[i].uri);
+    lib = isolate->object_store()->bootstrap_library(id);
+    ASSERT(lib.raw() == Library::LookupLibrary(thread, uri));
+    if (lib.IsNull()) {
+      lib = Library::NewLibraryHelper(uri, false);
+      lib.SetLoadRequested();
+      lib.Register(thread);
+      isolate->object_store()->set_bootstrap_library(id, lib);
+    }
+  }
+
+  return (kernel_buffer == NULL)
+             ? BootstrapFromSource(thread)
+             : BootstrapFromKernel(thread, kernel_buffer, kernel_buffer_length);
+}
+
 }  // namespace dart