Enable generation of instruction buffer on precompilation

runtime/bin/gen_snapshot.cc

Index: runtime/bin/gen_snapshot.cc
diff --git a/runtime/bin/gen_snapshot.cc b/runtime/bin/gen_snapshot.cc
index bb6d4f8502e425ff0748b4f84f39ead591311132..02e64267b1a283012b7448e5e68b36d5ccd0039b 100644
--- a/runtime/bin/gen_snapshot.cc
+++ b/runtime/bin/gen_snapshot.cc
@@ -9,6 +9,8 @@
 #include <string.h>
 #include <stdio.h>
 
+#include <cstdarg>
+
 #include "include/dart_api.h"
 
 #include "bin/builtin.h"
@@ -38,6 +40,8 @@ namespace bin {
 // if so which file to write the snapshot into.
 static const char* vm_isolate_snapshot_filename = NULL;
 static const char* isolate_snapshot_filename = NULL;
+static const char* instructions_snapshot_filename = NULL;
+static const char* embedder_entry_points_manifest = NULL;
 static const char* package_root = NULL;
 
 
@@ -88,6 +92,26 @@ static bool ProcessIsolateSnapshotOption(const char* option) {
 }
 
 
+static bool ProcessInstructionsSnapshotOption(const char* option) {
+  const char* name = ProcessOption(option, "--instructions_snapshot=");
+  if (name != NULL) {
+    instructions_snapshot_filename = name;
+    return true;
+  }
+  return false;
+}
+
+
+static bool ProcessEmbedderEntryPointsManifestOption(const char* option) {
+  const char* name = ProcessOption(option, "--embedder_entry_points_manifest=");
+  if (name != NULL) {
+    embedder_entry_points_manifest = name;
+    return true;
+  }
+  return false;
+}
+
+
 static bool ProcessPackageRootOption(const char* option) {
   const char* name = ProcessOption(option, "--package_root=");
   if (name != NULL) {
@@ -127,6 +151,8 @@ static int ParseArguments(int argc,
   while ((i < argc) && IsValidFlag(argv[i], kPrefix, kPrefixLen)) {
     if (ProcessVmIsolateSnapshotOption(argv[i]) ||
         ProcessIsolateSnapshotOption(argv[i]) ||
+        ProcessInstructionsSnapshotOption(argv[i]) ||
+        ProcessEmbedderEntryPointsManifestOption(argv[i]) ||
         ProcessURLmappingOption(argv[i]) ||
         ProcessPackageRootOption(argv[i])) {
       i += 1;
@@ -154,10 +180,32 @@ static int ParseArguments(int argc,
     return -1;
   }
 
+  if (instructions_snapshot_filename != NULL &&
+      embedder_entry_points_manifest == NULL) {
+    Log::PrintErr(
+        "Specifying an instructions snapshot filename indicates precompilation"
+        ". But no embedder entry points manifest was specified.\n\n");
+    return -1;
+  }
+
+  if (embedder_entry_points_manifest != NULL &&
+      instructions_snapshot_filename == NULL) {
+    Log::PrintErr(
+        "Specifying the embedder entry points manifest indicates "
+        "precompilation. But no instuctions snapshot was specified.\n\n");
+    return -1;
+  }
+
   return 0;
 }
 
 
+static bool IsSnapshottingForPrecompilation(void) {
+  return embedder_entry_points_manifest != NULL &&
+         instructions_snapshot_filename != NULL;
+}
+
+
 static void WriteSnapshotFile(const char* filename,
                               const uint8_t* buffer,
                               const intptr_t size) {
@@ -420,27 +468,54 @@ static Dart_Handle LoadGenericSnapshotCreationScript(
 
 
 static void PrintUsage() {
-  Log::PrintErr(
-"Usage:\n"
-"\n"
-"  gen_snapshot [<vm-flags>] [<options>] \\\n"
-"               --snapshot=<out-file> [<dart-script-file>]\n"
-"\n"
-"  Writes a snapshot of <dart-script-file> to <out-file>. If no\n"
-"  <dart-script-file> is passed, a generic snapshot of all the corelibs is\n"
-"  created. It is required to specify an output file name:\n"
-"\n"
-"    --snapshot=<file>          Generates a complete snapshot. Uses the url\n"
-"                               mapping specified on the command line to load\n"
-"                               the libraries.\n"
-"Supported options:\n"
-"\n"
-"--package_root=<path>\n"
-"  Where to find packages, that is, \"package:...\" imports.\n"
-"\n"
-"--url_mapping=<mapping>\n"
-"  Uses the URL mapping(s) specified on the command line to load the\n"
-"  libraries. For use only with --snapshot=.\n");
+#define STRINGERIZE(...) #__VA_ARGS__
+  // clang-format off
+  Log::PrintErr(STRINGERIZE(
+Usage:                                                                        \n
+  gen_snapshot [<vm-flags>] [<options>] [<dart-script-file>]                  \n
+                                                                              \n
+  Writes a snapshot of <dart-script-file> to the specified snapshot files. If \n
+  no <dart-script-file> is passed, a generic snapshot of all the corelibs is  \n
+  created. It is required to specify the VM isolate snapshot and the isolate  \n
+  snapshot. The other flags are related to precompilation and are optional.   \n
+                                                                              \n
+  Precompilation:                                                             \n
+  In order to configure the snapshotter for precompilation, both the          \n
+  instructions snapshot and embedder entry points manifest must be specified. \n
+  Machine code for the target architecture will be dumped into the            \n
+  instructions snapshot. This must be linked into the target binary in a      \n
+  separate step. The embedder entry points manifest lists the standalone entry\n
+  points into the VM. Not specifying these will cause the tree shaker to      \n
+  disregard the same as being used. The format of this manifest is as follows.\n
+  Each line in the manifest is a comma separated list of three elements. The  \n
+  first entry is the library URI, the second entry is the class name and the  \n
+  final entry the function name. The file must be terminated with a newline   \n
+  charater.                                                                   \n
+                                                                              \n
+    Example:                                                                  \n
+      dart:something,SomeClass,doSomething                                    \n
+                                                                              \n
+  Supported options:                                                          \n
+    --vm_isolate_snapshot=<file>      A full snapshot is a compact            \n
+    --isolate_snapshot=<file>         representation of the dart vm isolate   \n
+                                      heap and dart isolate heap states.      \n
+                                      Both these options are required         \n
+                                                                              \n
+    --package_root=<path>             Where to find packages, that is,        \n
+                                      package:...  imports.                   \n
+                                                                              \n
+    --url_mapping=<mapping>           Uses the URL mapping(s) specified on the\n
+                                      command line to load the libraries.     \n
+                                                                              \n
+    --instructions_snapshot=<file>    (Precompilation only) Contains the      \n
+                                      assembly that must be linked into       \n
+                                      the target binary                       \n
+                                                                              \n
+    --embedder_entry_points_manifest=<file> (Precompilation only) Contains the\n
+                                      stanalone embedder entry points         \n
+  ));
+  // clang-format on
+#undef STRINGERIZE
 }
 
 
@@ -456,7 +531,326 @@ static void VerifyLoaded(Dart_Handle library) {
 }
 
 
+static const char StubNativeFunctionName[] = "StubNativeFunction";
+
+
+void StubNativeFunction(Dart_NativeArguments arguments) {
+  // This is a stub function for the resolver
+  UNREACHABLE();
+}
+
+
+static Dart_NativeFunction StubNativeLookup(Dart_Handle name,
+                                            int argument_count,
+                                            bool* auto_setup_scope) {
+  return &StubNativeFunction;
+}
+
+
+static const uint8_t* StubNativeSymbol(Dart_NativeFunction nf) {
+  return reinterpret_cast<const uint8_t *>(StubNativeFunctionName);
+}
+
+
+static void SetupStubNativeResolver(size_t lib_index,
+                                    const Dart_QualifiedFunctionName* entry) {
+  // TODO(24686): Remove this.
+  Dart_Handle library_string = Dart_NewStringFromCString(entry->library_uri);
+  DART_CHECK_VALID(library_string);
+  Dart_Handle library = Dart_LookupLibrary(library_string);
+  // Embedder entry points may be setup in libraries that have not been
+  // explicitly loaded by the application script. In such cases, library lookup
+  // will fail. Manually load those libraries.
+  if (Dart_IsError(library)) {
+    static const uint32_t kLoadBufferMaxSize = 128;
+    char* load_buffer =
+        reinterpret_cast<char*>(calloc(kLoadBufferMaxSize, sizeof(char)));
+    snprintf(load_buffer,
+             kLoadBufferMaxSize,
+             "import '%s';",
+             DartUtils::GetStringValue(library_string));
+    Dart_Handle script_handle = Dart_NewStringFromCString(load_buffer);
+    memset(load_buffer, 0, kLoadBufferMaxSize);
+    snprintf(load_buffer,
+             kLoadBufferMaxSize,
+             "dart:_snapshot_%zu",
+             lib_index);
+    Dart_Handle script_url = Dart_NewStringFromCString(load_buffer);
+    free(load_buffer);
+    Dart_Handle loaded = Dart_LoadLibrary(script_url, script_handle, 0, 0);
+    DART_CHECK_VALID(loaded);
+
+    // Do a fresh lookup
+    library = Dart_LookupLibrary(library_string);
+  }
+
+  DART_CHECK_VALID(library);
+  Dart_Handle result =  Dart_SetNativeResolver(library,
+                                               &StubNativeLookup,
+                                               &StubNativeSymbol);
+  DART_CHECK_VALID(result);
+}
+
+
+static void ImportNativeEntryPointLibrariesIntoRoot(
+    const Dart_QualifiedFunctionName* entries) {
+  if (entries == NULL) {
+    return;
+  }
+
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == NULL) {
+      // The termination sentinel has null members.
+      break;
+    }
+    Dart_Handle entry_library =
+        Dart_LookupLibrary(Dart_NewStringFromCString(entry.library_uri));
+    DART_CHECK_VALID(entry_library);
+    Dart_Handle import_result = Dart_LibraryImportLibrary(
+        entry_library, Dart_RootLibrary(), Dart_EmptyString());
+    DART_CHECK_VALID(import_result);
+  }
+}
+
+
+static void SetupStubNativeResolversForPrecompilation(
+    const Dart_QualifiedFunctionName* entries) {
+
+  if (entries == NULL) {
+    return;
+  }
+
+  // Setup native resolvers for all libraries found in the manifest.
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == NULL) {
+      // The termination sentinel has null members.
+      break;
+    }
+    // Setup stub resolvers on loaded libraries
+    SetupStubNativeResolver(index, &entry);
+  }
+}
+
+
+static void CleanupEntryPointItem(const Dart_QualifiedFunctionName *entry) {
+  if (entry == NULL) {
+    return;
+  }
+  // The allocation used for these entries is zero'ed. So even in errors cases
+  // references to some entries will be null. Calling this on an already cleaned
+  // up entry is programmer error.
+  free(const_cast<char*>(entry->library_uri));
+  free(const_cast<char*>(entry->class_name));
+  free(const_cast<char*>(entry->function_name));
+}
+
+
+static void CleanupEntryPointsCollection(Dart_QualifiedFunctionName* entries) {
+  if (entries == NULL) {
+    return;
+  }
+
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == NULL) {
+      break;
+    }
+    CleanupEntryPointItem(&entry);
+  }
+  free(entries);
+}
+
+
+char *ParserErrorStringCreate(const char *format, ...) {

[rmacnak, 2015/10/22 16:46:15]

char* Parser
const char* format

[Chinmay, 2015/10/22 19:01:29]

ack

+  static const size_t kErrorBufferSize = 256;
+
+  char* error_buffer =
+      reinterpret_cast<char*>(calloc(kErrorBufferSize, sizeof(char)));
+  va_list args;
+  va_start(args, format);
+  vsnprintf(error_buffer, kErrorBufferSize, format, args);
+  va_end(args);
+
+  // In case of error, the buffer is released by the caller
+  return error_buffer;
+}
+
+
+const char* ParseEntryNameForIndex(uint8_t index) {
+  switch (index) {
+    case 0:
+      return "Library";
+    case 1:
+      return "Class";
+    case 2:
+      return "Function";
+    default:
+      return "Unknown";
+  }
+  return NULL;
+}
+
+
+static bool ParseEntryPointsManifestSingleLine(
+    const char* line, Dart_QualifiedFunctionName* entry, char** error) {
+  bool success = true;
+  size_t offset = 0;
+  for (uint8_t i = 0; i < 3; i++) {
+    const char* component = strchr(line + offset, i == 2 ? '\n' : ',');
+    if (component == NULL) {
+      success = false;
+      *error = ParserErrorStringCreate(
+          "Manifest entries must be comma separated and newline terminated. "
+          "Could not parse '%s' on line '%s'",
+          ParseEntryNameForIndex(i), line);
+      break;
+    }
+
+    int64_t chars_read = component - (line + offset);
+    if (chars_read <= 0) {
+      success = false;
+      *error =
+          ParserErrorStringCreate("There is no '%s' specified on line '%s'",
+                                  ParseEntryNameForIndex(i), line);
+      break;
+    }
+
+    if (entry != NULL) {
+      // These allocations are collected in |CleanupEntryPointsCollection|.
+      char* entry_item =
+          reinterpret_cast<char*>(calloc(chars_read + 1, sizeof(char)));
+      memcpy(entry_item, line + offset, chars_read);
+
+      switch (i) {
+        case 0:  // library
+          entry->library_uri = entry_item;
+          break;
+        case 1:  // class
+          entry->class_name = entry_item;
+          break;
+        case 2:  // function
+          entry->function_name = entry_item;
+          break;
+        default:
+          free(entry_item);
+          success = false;
+          *error = ParserErrorStringCreate("Internal parser error\n");
+          break;
+      }
+    }
+
+    offset += chars_read + 1;
+  }
+  return success;
+}
+
+
+int64_t ParseEntryPointsManifestLines(FILE* file,
+                                      Dart_QualifiedFunctionName* collection) {
+  int64_t entries = 0;
+
+  static const int kManifestMaxLineLength = 1024;
+  char* line = reinterpret_cast<char*>(malloc(kManifestMaxLineLength));
+  size_t line_number = 0;
+  while (true) {
+    line_number++;
+    char* read_line = fgets(line, kManifestMaxLineLength, file);
+
+    if (read_line == NULL) {
+      if ((feof(file) != 0) && (ferror(file) != 0)) {
+        Log::PrintErr(
+            "Error while reading line number %zu. The manifest must be "
+            "terminated by a newline\n",
+            line_number);
+        entries = -1;
+      }
+      break;
+    }
+
+    Dart_QualifiedFunctionName* entry =
+        collection != NULL ? collection + entries : NULL;
+
+    char *error_buffer = NULL;

[rmacnak, 2015/10/22 16:46:15]

char* error_buffer

[Chinmay, 2015/10/22 19:01:29]

ack

+    if (!ParseEntryPointsManifestSingleLine(read_line, entry, &error_buffer)) {
+      CleanupEntryPointItem(entry);
+      Log::PrintErr("Parser error on line %zu: %s\n", line_number,
+                    error_buffer);
+      free(error_buffer);
+      entries = -1;
+      break;
+    }
+
+    entries++;
+  }
+
+  free(line);
+
+  return entries;
+}
+
+
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestFile(
+    const char* path) {
+  if (path == NULL) {
+    return NULL;
+  }
+
+  FILE* file = fopen(path, "r");

[rmacnak, 2015/10/22 16:46:15]

Are these IO functions portable? gen_snapshot needs to keep working on Windows.
It's better to use the functions from runtime/bin/file.h instead like
WriteSnapshotFile does.

[Chinmay, 2015/10/22 19:01:29]

I am quite certain I have used portable functions. I could not find functions to
check the stream status in bin/file.h. So had to resort to this implementation.

+
+  if (file == NULL) {
+    Log::PrintErr("Could not open entry points manifest file\n");
+    return NULL;
+  }
+
+  // Parse the file once but don't store the results. This is done to first
+  // determine the number of entries in the manifest
+  int64_t entry_count = ParseEntryPointsManifestLines(file, NULL);
+
+  if (entry_count <= 0) {
+    Log::PrintErr(
+        "Manifest file specified is invalid or contained no entries\n");
+    fclose(file);
+    return NULL;
+  }
+
+  rewind(file);
+
+  // Allocate enough storage for the entries in the file plus a termination
+  // sentinel and parse it again to populate the allocation
+  Dart_QualifiedFunctionName* entries =
+      reinterpret_cast<Dart_QualifiedFunctionName*>(
+          calloc(entry_count + 1, sizeof(Dart_QualifiedFunctionName)));
+
+  int64_t parsed_entry_count = ParseEntryPointsManifestLines(file, entries);
+  ASSERT(parsed_entry_count == entry_count);
+
+  fclose(file);
+
+  // The entries allocation must be explicitly cleaned up via
+  // |CleanupEntryPointsCollection|
+  return entries;
+}
+
+
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestIfPresent() {
+  Dart_QualifiedFunctionName* entries =
+      ParseEntryPointsManifestFile(embedder_entry_points_manifest);
+  if (entries == NULL && IsSnapshottingForPrecompilation()) {
+    Log::PrintErr(
+        "Could not find native embedder entry points during precompilation\n");
+    exit(255);
+  }
+  return entries;
+}
+
+
 static void CreateAndWriteSnapshot() {
+  ASSERT(!IsSnapshottingForPrecompilation());
   Dart_Handle result;
   uint8_t* vm_isolate_buffer = NULL;
   intptr_t vm_isolate_size = 0;
@@ -485,6 +879,49 @@ static void CreateAndWriteSnapshot() {
 }
 
 
+static void CreateAndWritePrecompiledSnapshot(
+    Dart_QualifiedFunctionName* standalone_entry_points) {
+  ASSERT(IsSnapshottingForPrecompilation());
+  Dart_Handle result;
+  uint8_t* vm_isolate_buffer = NULL;
+  intptr_t vm_isolate_size = 0;
+  uint8_t* isolate_buffer = NULL;
+  intptr_t isolate_size = 0;
+  uint8_t* instructions_buffer = NULL;
+  intptr_t instructions_size = 0;
+
+  // Precompile with specified embedder entry points
+  result = Dart_Precompile(standalone_entry_points, true);
+  CHECK_RESULT(result);
+
+  // Create a precompiled snapshot. This gives us an instruction buffer with
+  // machine code
+  result = Dart_CreatePrecompiledSnapshot(&vm_isolate_buffer,
+                                          &vm_isolate_size,
+                                          &isolate_buffer,
+                                          &isolate_size,
+                                          &instructions_buffer,
+                                          &instructions_size);
+  CHECK_RESULT(result);
+
+  // Now write the vm isolate, isolate and instructions snapshots out to the
+  // specified files and exit.
+  WriteSnapshotFile(vm_isolate_snapshot_filename,
+                    vm_isolate_buffer,
+                    vm_isolate_size);
+  WriteSnapshotFile(isolate_snapshot_filename,
+                    isolate_buffer,
+                    isolate_size);
+  WriteSnapshotFile(instructions_snapshot_filename,
+                    instructions_buffer,
+                    instructions_size);
+  Dart_ExitScope();
+
+  // Shutdown the isolate.
+  Dart_ShutdownIsolate();
+}
+
+
 static void SetupForUriResolution() {
   // Set up the library tag handler for this isolate.
   Dart_Handle result = Dart_SetLibraryTagHandler(DartUtils::LibraryTagHandler);
@@ -589,6 +1026,11 @@ int main(int argc, char** argv) {
   // Workaround until issue 21620 is fixed.
   // (https://github.com/dart-lang/sdk/issues/21620)
   vm_options.AddArgument("--no-concurrent_sweep");
+
+  if (IsSnapshottingForPrecompilation()) {
+    vm_options.AddArgument("--precompilation");
+  }
+
   Dart_SetVMFlags(vm_options.count(), vm_options.arguments());
 
   // Initialize the Dart VM.
@@ -672,13 +1114,30 @@ int main(int argc, char** argv) {
     // URL mapping specified on the command line to load the libraries.
     result = Dart_SetLibraryTagHandler(CreateSnapshotLibraryTagHandler);
     CHECK_RESULT(result);
+
+    Dart_QualifiedFunctionName* entry_points =
+        ParseEntryPointsManifestIfPresent();
+
+    SetupStubNativeResolversForPrecompilation(entry_points);
+
     // Load the specified script.
     library = LoadSnapshotCreationScript(app_script_name);
     VerifyLoaded(library);
+
+    ImportNativeEntryPointLibrariesIntoRoot(entry_points);
+
     // Ensure that we mark all libraries as loaded.
     result = Dart_FinalizeLoading(false);
     CHECK_RESULT(result);
-    CreateAndWriteSnapshot();
+
+    if (entry_points == NULL) {
+      ASSERT(!IsSnapshottingForPrecompilation());
+      CreateAndWriteSnapshot();
+    } else {
+      CreateAndWritePrecompiledSnapshot(entry_points);
+    }
+
+    CleanupEntryPointsCollection(entry_points);
 
     Dart_EnterIsolate(UriResolverIsolateScope::isolate);
     Dart_ShutdownIsolate();