Enable generation of instruction buffer on precompilation

runtime/bin/gen_snapshot.cc

Index: runtime/bin/gen_snapshot.cc
diff --git a/runtime/bin/gen_snapshot.cc b/runtime/bin/gen_snapshot.cc
index bb6d4f8502e425ff0748b4f84f39ead591311132..1d696ba50969b3da91ec7bc5e09452dd3f727f95 100644
--- a/runtime/bin/gen_snapshot.cc
+++ b/runtime/bin/gen_snapshot.cc
@@ -38,6 +38,8 @@ namespace bin {
 // if so which file to write the snapshot into.
 static const char* vm_isolate_snapshot_filename = NULL;
 static const char* isolate_snapshot_filename = NULL;
+static const char* instructions_snapshot_filename = NULL;
+static const char* vm_entry_points_manifest = NULL;

[rmacnak, 2015/10/21 23:06:18]

Throughout: s/vm_entry_points/embedder_entry_points/

There is a different list in the precompiler for the VM's own entry points.

[Chinmay, 2015/10/22 00:13:16]

ack

 static const char* package_root = NULL;
 
 
@@ -88,6 +90,26 @@ static bool ProcessIsolateSnapshotOption(const char* option) {
 }
 
 
+static bool ProcessInstructionsSnapshotOption(const char* option) {
+  const char* name = ProcessOption(option, "--instructions_snapshot=");
+  if (name != NULL) {
+    instructions_snapshot_filename = name;
+    return true;
+  }
+  return false;
+}
+
+
+static bool ProcessVMEntryPointsManifestOption(const char* option) {
+  const char* name = ProcessOption(option, "--vm_entry_points_manifest=");
+  if (name != NULL) {
+    vm_entry_points_manifest = name;
+    return true;
+  }
+  return false;
+}
+
+
 static bool ProcessPackageRootOption(const char* option) {
   const char* name = ProcessOption(option, "--package_root=");
   if (name != NULL) {
@@ -127,6 +149,8 @@ static int ParseArguments(int argc,
   while ((i < argc) && IsValidFlag(argv[i], kPrefix, kPrefixLen)) {
     if (ProcessVmIsolateSnapshotOption(argv[i]) ||
         ProcessIsolateSnapshotOption(argv[i]) ||
+        ProcessInstructionsSnapshotOption(argv[i]) ||
+        ProcessVMEntryPointsManifestOption(argv[i]) ||
         ProcessURLmappingOption(argv[i]) ||
         ProcessPackageRootOption(argv[i])) {
       i += 1;
@@ -154,10 +178,32 @@ static int ParseArguments(int argc,
     return -1;
   }
 
+  if (instructions_snapshot_filename != NULL &&
+      vm_entry_points_manifest == NULL) {
+    Log::PrintErr(
+        "Specifying an instructions snapshot filename indicates "
+        "precompilation. But no VM entry points manifest was specified.\n\n");
+    return -1;
+  }
+
+  if (vm_entry_points_manifest != NULL &&
+      instructions_snapshot_filename == NULL) {
+    Log::PrintErr(
+        "Specifying the VM entry points manifest indicates precompilation. But "
+        "no instuctions snapshot was specified.\n\n");
+    return -1;
+  }
+
   return 0;
 }
 
 
+static bool IsSnapshottingForPrecompilation(void) {
+  return vm_entry_points_manifest != NULL &&
+         instructions_snapshot_filename != NULL;
+}
+
+
 static void WriteSnapshotFile(const char* filename,
                               const uint8_t* buffer,
                               const intptr_t size) {
@@ -420,27 +466,54 @@ static Dart_Handle LoadGenericSnapshotCreationScript(
 
 
 static void PrintUsage() {
-  Log::PrintErr(
-"Usage:\n"
-"\n"
-"  gen_snapshot [<vm-flags>] [<options>] \\\n"
-"               --snapshot=<out-file> [<dart-script-file>]\n"
-"\n"
-"  Writes a snapshot of <dart-script-file> to <out-file>. If no\n"
-"  <dart-script-file> is passed, a generic snapshot of all the corelibs is\n"
-"  created. It is required to specify an output file name:\n"
-"\n"
-"    --snapshot=<file>          Generates a complete snapshot. Uses the url\n"
-"                               mapping specified on the command line to load\n"
-"                               the libraries.\n"
-"Supported options:\n"
-"\n"
-"--package_root=<path>\n"
-"  Where to find packages, that is, \"package:...\" imports.\n"
-"\n"
-"--url_mapping=<mapping>\n"
-"  Uses the URL mapping(s) specified on the command line to load the\n"
-"  libraries. For use only with --snapshot=.\n");
+#define STRINGERIZE(...) #__VA_ARGS__
+  // clang-format off
+  Log::PrintErr(STRINGERIZE(
+Usage:                                                                        \n
+  gen_snapshot [<vm-flags>] [<options>] [<dart-script-file>]                  \n
+                                                                              \n
+  Writes a snapshot of <dart-script-file> to the specified snapshot files. If \n
+  no <dart-script-file> is passed, a generic snapshot of all the corelibs is  \n
+  created. It is required to specify the VM isolate snapshot and the isolate  \n
+  snapshot. The other flags are related to precompilation and are optional.   \n
+                                                                              \n
+  Precompilation:                                                             \n
+  In order to configure the snapshotter for precompilation, both the          \n
+  instructions snapshot and the VM entry points manifest must be specified.   \n
+  Machine code for the target architecture will be dumped into the            \n
+  instructions snapshot. This must be linked into the target binary in a      \n
+  separate step. The VM entry points manifest lists the standalone entry      \n
+  points into the VM. Not specifying these will cause the tree shaker to      \n
+  disregard the same as being used. The format of this manifest is as follows.\n
+  Each line in the manifest is a comma separated list of three elements. The  \n
+  first entry is the library URI, the second entry is the class name and the  \n
+  final entry the function name. The file must be terminated with a newline   \n
+  charater.                                                                   \n
+                                                                              \n
+    Example:                                                                  \n
+      dart:something,SomeClass,doSomething                                    \n
+                                                                              \n
+  Supported options:                                                          \n
+    --vm_isolate_snapshot=<file>      A full snapshot is a compact            \n
+    --isolate_snapshot=<file>         representation of the dart vm isolate   \n
+                                      heap and dart isolate heap states.      \n
+                                      Both these options are required         \n
+                                                                              \n
+    --package_root=<path>             Where to find packages, that is,        \n
+                                      package:...  imports.                   \n
+                                                                              \n
+    --url_mapping=<mapping>           Uses the URL mapping(s) specified on the\n
+                                      command line to load the libraries.     \n
+                                                                              \n
+    --instructions_snapshot=<file>    (Precompilation only) Contains the      \n
+                                      machine code that must be linked into   \n

[rmacnak, 2015/10/21 23:06:18]

s/machine code/assembly

[Chinmay, 2015/10/22 00:13:16]

ack

+                                      the target binary                       \n
+                                                                              \n
+    --vm_entry_points_manifest=<file> (Precompilation only) Contains the      \n
+                                      stanalone VM entry points               \n
+  ));
+  // clang-format on
+#undef STRINGERIZE
 }
 
 
@@ -456,6 +529,268 @@ static void VerifyLoaded(Dart_Handle library) {
 }
 
 
+static const char StubNativeFunctionName[] = "StubNativeFunction";
+
+
+void StubNativeFunction(Dart_NativeArguments arguments) {
+  // This is a stub function for the resolver
+  ASSERT(false);

[rmacnak, 2015/10/21 23:06:18]

UNREACHABLE();

[Chinmay, 2015/10/22 00:13:16]

ack

+}
+
+
+static Dart_NativeFunction StubNativeLookup(Dart_Handle name,
+                                     int argument_count,

[Cutch, 2015/10/21 22:00:08]

align arguments

[Chinmay, 2015/10/22 00:13:17]

ack

+                                     bool* auto_setup_scope) {
+  return &StubNativeFunction;
+}
+
+
+static const uint8_t* StubNativeSymbol(Dart_NativeFunction nf) {
+  return reinterpret_cast<const uint8_t *>(StubNativeFunctionName);
+}
+
+
+static void SetupStubNativeResolver(size_t lib_index,

[rmacnak, 2015/10/21 23:06:18]

TODO(24686): Remove this.

[Chinmay, 2015/10/22 00:13:16]

ack

+                                    const Dart_QualifiedFunctionName* entry) {
+  Dart_Handle library_string = Dart_NewStringFromCString(entry->library_uri);
+  DART_CHECK_VALID(library_string);
+  Dart_Handle library = Dart_LookupLibrary(library_string);
+  // VM entry points may be setup in libraries that have not been explicitly
+  // loaded by the application script. In such cases, library lookup will
+  // fail. Manually load those libraries.
+  if (Dart_IsError(library)) {
+    static const uint32_t kLoadBufferMaxSize = 128;
+    char* load_buffer =
+        reinterpret_cast<char*>(calloc(kLoadBufferMaxSize, sizeof(char)));
+    snprintf(load_buffer,
+             kLoadBufferMaxSize,
+             "import '%s';",
+             DartUtils::GetStringValue(library_string));
+    Dart_Handle script_handle = Dart_NewStringFromCString(load_buffer);
+    memset(load_buffer, 0, kLoadBufferMaxSize);
+    snprintf(load_buffer,
+             kLoadBufferMaxSize,
+             "dart:_snapshot_%zu",
+             lib_index);
+    Dart_Handle script_url = Dart_NewStringFromCString(load_buffer);
+    free(load_buffer);
+    Dart_Handle loaded = Dart_LoadLibrary(script_url, script_handle, 0, 0);
+    DART_CHECK_VALID(loaded);
+
+    // Do a fresh lookup
+    library = Dart_LookupLibrary(library_string);
+  }
+
+  DART_CHECK_VALID(library);
+  Dart_Handle result =  Dart_SetNativeResolver(library,
+                                               &StubNativeLookup,
+                                               &StubNativeSymbol);
+  DART_CHECK_VALID(result);
+}
+
+
+static void ImportNativeEntryPointLibrariesIntoRoot(
+    const Dart_QualifiedFunctionName* entries) {
+  if (entries == nullptr) {

[rmacnak, 2015/10/21 23:06:18]

Here and throughout s/nullptr/NULL

nullptr is undefined in the standalone builds

[Chinmay, 2015/10/22 00:13:17]

ack

+    return;
+  }
+
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == nullptr) {
+      // The termination sentinel has null members.
+      break;
+    }
+    Dart_Handle entry_library =
+        Dart_LookupLibrary(Dart_NewStringFromCString(entry.library_uri));
+    DART_CHECK_VALID(entry_library);
+    Dart_Handle import_result = Dart_LibraryImportLibrary(
+        entry_library, Dart_RootLibrary(), Dart_EmptyString());
+    DART_CHECK_VALID(import_result);
+  }
+}
+
+
+static void SetupStubNativeResolversForPrecompilation(
+    const Dart_QualifiedFunctionName* entries) {
+
+  if (entries == nullptr) {
+    return;
+  }
+
+  // Setup native resolvers for all libraries found in the manifest.
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == nullptr) {
+      // The termination sentinel has null members.
+      break;
+    }
+    // Setup stub resolvers on loaded libraries
+    SetupStubNativeResolver(index, &entry);
+  }
+}
+
+
+static bool ParseEntryPointsManifestSingleLine(

[Cutch, 2015/10/21 22:00:08]

This should take an char** error which indicates what went wrong on failure
which can be printed by the caller.

[Chinmay, 2015/10/22 00:13:16]

ack

+    const char* line, Dart_QualifiedFunctionName* entry) {
+  bool success = true;
+  size_t offset = 0;
+  for (uint8_t i = 0; i < 3; i++) {
+    char* component = strchr(line + offset, i == 2 ? '\n' : ',');

[rmacnak, 2015/10/21 23:06:18]

const char* component

[Chinmay, 2015/10/22 00:13:16]

ack

+    if (component == nullptr) {
+      success = false;
+      break;
+    }
+
+    int64_t chars_read = component - (line + offset);
+    if (chars_read <= 0) {
+      success = false;
+      break;
+    }
+
+    if (entry != nullptr) {
+      // These allocations are collected in |CleanupEntryPointsCollection|.
+      char* entry_item =
+          reinterpret_cast<char*>(calloc(chars_read + 1, sizeof(char)));
+      memcpy(entry_item, line + offset, chars_read);
+
+      switch (i) {
+        case 0:  // library
+          entry->library_uri = entry_item;
+          break;
+        case 1:  // class
+          entry->class_name = entry_item;
+          break;
+        case 2:  // function
+          entry->function_name = entry_item;
+          break;
+        default:
+          free(entry_item);
+          success = false;
+          break;
+      }
+    }
+
+    offset += chars_read + 1;
+  }
+  return success;
+}
+
+
+int64_t ParseEntryPointsManifestLines(FILE* file,
+                                      Dart_QualifiedFunctionName* collection) {
+  int64_t entries = 0;
+
+  static const int kManifestMaxLineLength = 1024;
+  char* line = reinterpret_cast<char*>(malloc(kManifestMaxLineLength));
+
+  while (true) {
+    char* read_line = fgets(line, kManifestMaxLineLength, file);
+
+    if (read_line == nullptr) {
+      if ((feof(file) != 0) && (ferror(file) != 0)) {
+        entries = -1;
+      }
+      break;
+    }
+
+    Dart_QualifiedFunctionName* entry =
+        collection != nullptr ? collection + entries : nullptr;
+
+    if (!ParseEntryPointsManifestSingleLine(read_line, entry)) {
+      entries = -1;

[Cutch, 2015/10/21 22:00:08]

This might leak some |entry_item| allocated above in the error case.

[Chinmay, 2015/10/22 00:13:16]

ack

+      break;
+    }
+
+    entries++;
+  }
+
+  free(line);
+
+  return entries;
+}
+
+
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestFile(
+    const char* path) {
+  if (path == nullptr) {
+    return nullptr;
+  }
+
+  FILE* file = fopen(path, "r");
+
+  if (file == nullptr) {
+    Log::PrintErr("Could not open entry points manifest file\n");
+    return nullptr;
+  }
+
+  // Parse the file once but don't store the results. This is done to first
+  // determine the number of entries in the manifest
+  int64_t entry_count = ParseEntryPointsManifestLines(file, nullptr);
+
+  if (entry_count <= 0) {
+    Log::PrintErr("Manifest file specified is invalid or contained no entries");
+    fclose(file);
+    return nullptr;
+  }
+
+  rewind(file);
+
+  // Allocate enough storage for the entries in the file plus a termination
+  // sentinel and parse it again to populate the allocation
+  Dart_QualifiedFunctionName* entries =
+      reinterpret_cast<Dart_QualifiedFunctionName*>(
+          calloc(entry_count + 1, sizeof(Dart_QualifiedFunctionName)));
+
+  int64_t parsed_entry_count = ParseEntryPointsManifestLines(file, entries);
+
+  if (parsed_entry_count != entry_count) {

[Cutch, 2015/10/21 22:00:08]

You're calling the same function twice with the same inputs. How could this
happen?

[Chinmay, 2015/10/22 00:13:17]

Was being paranoid since I am reading from a file on disk. Will get rid of it
and add an assert instead.

+    free(entries);

[Cutch, 2015/10/21 22:00:08]

This leaks pointers inside of |entries|. Call CleanupEntryPointsCollection
instead of just free.

[Chinmay, 2015/10/22 00:13:17]

ack. Removed by the assert.

+    fclose(file);
+    return nullptr;
+  }
+
+  fclose(file);
+
+  // The entries allocation must be explicitly cleaned up via
+  // |CleanupEntryPointsCollection|
+  return entries;
+}
+
+
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestIfPresent() {
+  Dart_QualifiedFunctionName* entries =
+      ParseEntryPointsManifestFile(vm_entry_points_manifest);
+  if (entries == nullptr && IsSnapshottingForPrecompilation()) {

[Cutch, 2015/10/21 22:00:08]

s/nullptr/NULL here and elsewhere

[Chinmay, 2015/10/22 00:13:16]

ack

+    Log::PrintErr(
+        "Could not find native VM entry points during precompilation");
+    exit(255);
+  }
+  return entries;
+}
+
+
+void CleanupEntryPointsCollection(Dart_QualifiedFunctionName* entries) {
+  if (entries == nullptr) {
+    return;
+  }
+
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == nullptr) {

[Cutch, 2015/10/21 22:00:08]

s/nullptr/NULL here and elsewhere

[Chinmay, 2015/10/22 00:13:16]

ack

+      break;
+    }
+    free((char*)entry.library_uri);

[Cutch, 2015/10/21 22:00:08]

use c++ casts

[Chinmay, 2015/10/22 00:13:17]

ack

+    free((char*)entry.class_name);
+    free((char*)entry.function_name);
+  }
+  free(entries);
+}
+
+
 static void CreateAndWriteSnapshot() {
   Dart_Handle result;
   uint8_t* vm_isolate_buffer = NULL;
@@ -485,6 +820,55 @@ static void CreateAndWriteSnapshot() {
 }
 
 
+static void CreateAndWritePrecompiledSnapshot(
+    Dart_QualifiedFunctionName* standalone_entry_points) {
+
+  if (standalone_entry_points == nullptr) {

[Cutch, 2015/10/21 22:00:08]

Can you hoist this out or rename it? It's confusing to see the call to
CreateAndWritePrecompiledSnapshot below when it's only writing a precompiled
snapshot sometimes.

[Chinmay, 2015/10/22 00:13:16]

ack

+    ASSERT(!IsSnapshottingForPrecompilation());
+    CreateAndWriteSnapshot();
+    return;
+  }
+
+  Dart_Handle result;
+  uint8_t* vm_isolate_buffer = NULL;
+  intptr_t vm_isolate_size = 0;
+  uint8_t* isolate_buffer = NULL;
+  intptr_t isolate_size = 0;
+  uint8_t* instructions_buffer = NULL;
+  intptr_t instructions_size = 0;
+
+  // Precompile with specified VM entry points
+  result = Dart_Precompile(standalone_entry_points, true);
+  CHECK_RESULT(result);
+
+  // Create a precompiled snapshot. This gives us an instruction buffer with
+  // machine code
+  result = Dart_CreatePrecompiledSnapshot(&vm_isolate_buffer,
+                                          &vm_isolate_size,
+                                          &isolate_buffer,
+                                          &isolate_size,
+                                          &instructions_buffer,
+                                          &instructions_size);
+  CHECK_RESULT(result);
+
+  // Now write the vm isolate, isolate and instructions snapshots out to the
+  // specified files and exit.
+  WriteSnapshotFile(vm_isolate_snapshot_filename,
+                    vm_isolate_buffer,
+                    vm_isolate_size);
+  WriteSnapshotFile(isolate_snapshot_filename,
+                    isolate_buffer,
+                    isolate_size);
+  WriteSnapshotFile(instructions_snapshot_filename,
+                    instructions_buffer,
+                    instructions_size);
+  Dart_ExitScope();
+
+  // Shutdown the isolate.
+  Dart_ShutdownIsolate();
+}
+
+
 static void SetupForUriResolution() {
   // Set up the library tag handler for this isolate.
   Dart_Handle result = Dart_SetLibraryTagHandler(DartUtils::LibraryTagHandler);
@@ -589,6 +973,12 @@ int main(int argc, char** argv) {
   // Workaround until issue 21620 is fixed.
   // (https://github.com/dart-lang/sdk/issues/21620)
   vm_options.AddArgument("--no-concurrent_sweep");
+
+  if (IsSnapshottingForPrecompilation()) {
+    vm_options.AddArgument("--precompilation");
+    vm_options.AddArgument("--write_protect_code=false");

[rmacnak, 2015/10/21 23:06:18]

--write_protect_code=false should be unnecessary now

[Chinmay, 2015/10/22 00:13:16]

ack

+  }
+
   Dart_SetVMFlags(vm_options.count(), vm_options.arguments());
 
   // Initialize the Dart VM.
@@ -672,13 +1062,24 @@ int main(int argc, char** argv) {
     // URL mapping specified on the command line to load the libraries.
     result = Dart_SetLibraryTagHandler(CreateSnapshotLibraryTagHandler);
     CHECK_RESULT(result);
+
+    Dart_QualifiedFunctionName* entry_points =
+        ParseEntryPointsManifestIfPresent();
+
+    SetupStubNativeResolversForPrecompilation(entry_points);
+
     // Load the specified script.
     library = LoadSnapshotCreationScript(app_script_name);
     VerifyLoaded(library);
+
+    ImportNativeEntryPointLibrariesIntoRoot(entry_points);
+
     // Ensure that we mark all libraries as loaded.
     result = Dart_FinalizeLoading(false);
     CHECK_RESULT(result);
-    CreateAndWriteSnapshot();
+    CreateAndWritePrecompiledSnapshot(entry_points);
+
+    CleanupEntryPointsCollection(entry_points);
 
     Dart_EnterIsolate(UriResolverIsolateScope::isolate);
     Dart_ShutdownIsolate();