Enable generation of instruction buffer on precompilation

runtime/bin/gen_snapshot.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 // Generate a snapshot file after loading all the scripts specified on the
 // command line.
 
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
 
+#include <cstdarg>
+
 #include "include/dart_api.h"
 
 #include "bin/builtin.h"
 #include "bin/dartutils.h"
 #include "bin/eventhandler.h"
 #include "bin/file.h"
 #include "bin/log.h"
 #include "bin/thread.h"
 #include "bin/vmservice_impl.h"
 
 #include "platform/globals.h"
 
 
 namespace dart {
 namespace bin {
 
 #define CHECK_RESULT(result)                                                   \
   if (Dart_IsError(result)) {                                                  \
     Log::PrintErr("Error: %s", Dart_GetError(result));                         \
     Dart_ExitScope();                                                          \
     Dart_ShutdownIsolate();                                                    \
     exit(255);                                                                 \
   }                                                                            \
 
 
 // Global state that indicates whether a snapshot is to be created and
 // if so which file to write the snapshot into.
 static const char* vm_isolate_snapshot_filename = NULL;
 static const char* isolate_snapshot_filename = NULL;
+static const char* instructions_snapshot_filename = NULL;
+static const char* embedder_entry_points_manifest = NULL;
 static const char* package_root = NULL;
 
 
 // Global state which contains a pointer to the script name for which
 // a snapshot needs to be created (NULL would result in the creation
 // of a generic snapshot that contains only the corelibs).
 static char* app_script_name = NULL;
 
 
 // Global state that captures the URL mappings specified on the command line.
@@ skipping 30 matching lines @@
 static bool ProcessIsolateSnapshotOption(const char* option) {
   const char* name = ProcessOption(option, "--isolate_snapshot=");
   if (name != NULL) {
     isolate_snapshot_filename = name;
     return true;
   }
   return false;
 }
 
 
+static bool ProcessInstructionsSnapshotOption(const char* option) {
+  const char* name = ProcessOption(option, "--instructions_snapshot=");
+  if (name != NULL) {
+    instructions_snapshot_filename = name;
+    return true;
+  }
+  return false;
+}
+
+
+static bool ProcessEmbedderEntryPointsManifestOption(const char* option) {
+  const char* name = ProcessOption(option, "--embedder_entry_points_manifest=");
+  if (name != NULL) {
+    embedder_entry_points_manifest = name;
+    return true;
+  }
+  return false;
+}
+
+
 static bool ProcessPackageRootOption(const char* option) {
   const char* name = ProcessOption(option, "--package_root=");
   if (name != NULL) {
     package_root = name;
     return true;
   }
   return false;
 }
 
 
@@ skipping 19 matching lines @@
   const char* kPrefix = "--";
   const intptr_t kPrefixLen = strlen(kPrefix);
 
   // Skip the binary name.
   int i = 1;
 
   // Parse out the vm options.
   while ((i < argc) && IsValidFlag(argv[i], kPrefix, kPrefixLen)) {
     if (ProcessVmIsolateSnapshotOption(argv[i]) ||
         ProcessIsolateSnapshotOption(argv[i]) ||
+        ProcessInstructionsSnapshotOption(argv[i]) ||
+        ProcessEmbedderEntryPointsManifestOption(argv[i]) ||
         ProcessURLmappingOption(argv[i]) ||
         ProcessPackageRootOption(argv[i])) {
       i += 1;
       continue;
     }
     vm_options->AddArgument(argv[i]);
     i += 1;
   }
 
   // Get the script name.
   if (i < argc) {
     *script_name = argv[i];
     i += 1;
   } else {
     *script_name = NULL;
   }
 
   if (vm_isolate_snapshot_filename == NULL) {
     Log::PrintErr("No vm isolate snapshot output file specified.\n\n");
     return -1;
   }
 
   if (isolate_snapshot_filename == NULL) {
     Log::PrintErr("No isolate snapshot output file specified.\n\n");
     return -1;
   }
 
+  if (instructions_snapshot_filename != NULL &&
+      embedder_entry_points_manifest == NULL) {
+    Log::PrintErr(
+        "Specifying an instructions snapshot filename indicates precompilation"
+        ". But no embedder entry points manifest was specified.\n\n");
+    return -1;
+  }
+
+  if (embedder_entry_points_manifest != NULL &&
+      instructions_snapshot_filename == NULL) {
+    Log::PrintErr(
+        "Specifying the embedder entry points manifest indicates "
+        "precompilation. But no instuctions snapshot was specified.\n\n");
+    return -1;
+  }
+
   return 0;
 }
 
 
+static bool IsSnapshottingForPrecompilation(void) {
+  return embedder_entry_points_manifest != NULL &&
+         instructions_snapshot_filename != NULL;
+}
+
+
 static void WriteSnapshotFile(const char* filename,
                               const uint8_t* buffer,
                               const intptr_t size) {
   File* file = File::Open(filename, File::kWriteTruncate);
   ASSERT(file != NULL);
   if (!file->WriteFully(buffer, size)) {
     Log::PrintErr("Error: Failed to write snapshot file.\n\n");
   }
   delete file;
 }
@@ skipping 242 matching lines @@
   Dart_Handle lib;
   // Load the builtin library to make it available in the snapshot
   // for importing.
   lib = Builtin::LoadAndCheckLibrary(id);
   ASSERT(!Dart_IsError(lib));
   return lib;
 }
 
 
 static void PrintUsage() {
-  Log::PrintErr(
-"Usage:\n"
-"\n"
-"  gen_snapshot [<vm-flags>] [<options>] \\\n"
-"               --snapshot=<out-file> [<dart-script-file>]\n"
-"\n"
-"  Writes a snapshot of <dart-script-file> to <out-file>. If no\n"
-"  <dart-script-file> is passed, a generic snapshot of all the corelibs is\n"
-"  created. It is required to specify an output file name:\n"
-"\n"
-"    --snapshot=<file>          Generates a complete snapshot. Uses the url\n"
-"                               mapping specified on the command line to load\n"
-"                               the libraries.\n"
-"Supported options:\n"
-"\n"
-"--package_root=<path>\n"
-"  Where to find packages, that is, \"package:...\" imports.\n"
-"\n"
-"--url_mapping=<mapping>\n"
-"  Uses the URL mapping(s) specified on the command line to load the\n"
-"  libraries. For use only with --snapshot=.\n");
+#define STRINGERIZE(...) #__VA_ARGS__
+  // clang-format off
+  Log::PrintErr(STRINGERIZE(
+Usage:                                                                        \n
+  gen_snapshot [<vm-flags>] [<options>] [<dart-script-file>]                  \n
+                                                                              \n
+  Writes a snapshot of <dart-script-file> to the specified snapshot files. If \n
+  no <dart-script-file> is passed, a generic snapshot of all the corelibs is  \n
+  created. It is required to specify the VM isolate snapshot and the isolate  \n
+  snapshot. The other flags are related to precompilation and are optional.   \n
+                                                                              \n
+  Precompilation:                                                             \n
+  In order to configure the snapshotter for precompilation, both the          \n
+  instructions snapshot and embedder entry points manifest must be specified. \n
+  Machine code for the target architecture will be dumped into the            \n
+  instructions snapshot. This must be linked into the target binary in a      \n
+  separate step. The embedder entry points manifest lists the standalone entry\n
+  points into the VM. Not specifying these will cause the tree shaker to      \n
+  disregard the same as being used. The format of this manifest is as follows.\n
+  Each line in the manifest is a comma separated list of three elements. The  \n
+  first entry is the library URI, the second entry is the class name and the  \n
+  final entry the function name. The file must be terminated with a newline   \n
+  charater.                                                                   \n
+                                                                              \n
+    Example:                                                                  \n
+      dart:something,SomeClass,doSomething                                    \n
+                                                                              \n
+  Supported options:                                                          \n
+    --vm_isolate_snapshot=<file>      A full snapshot is a compact            \n
+    --isolate_snapshot=<file>         representation of the dart vm isolate   \n
+                                      heap and dart isolate heap states.      \n
+                                      Both these options are required         \n
+                                                                              \n
+    --package_root=<path>             Where to find packages, that is,        \n
+                                      package:...  imports.                   \n
+                                                                              \n
+    --url_mapping=<mapping>           Uses the URL mapping(s) specified on the\n
+                                      command line to load the libraries.     \n
+                                                                              \n
+    --instructions_snapshot=<file>    (Precompilation only) Contains the      \n
+                                      assembly that must be linked into       \n
+                                      the target binary                       \n
+                                                                              \n
+    --embedder_entry_points_manifest=<file> (Precompilation only) Contains the\n
+                                      stanalone embedder entry points         \n
+  ));
+  // clang-format on
+#undef STRINGERIZE
 }
 
 
 static void VerifyLoaded(Dart_Handle library) {
   if (Dart_IsError(library)) {
     const char* err_msg = Dart_GetError(library);
     Log::PrintErr("Errors encountered while loading: %s\n", err_msg);
     Dart_ExitScope();
     Dart_ShutdownIsolate();
     exit(255);
   }
   ASSERT(Dart_IsLibrary(library));
 }
 
 
+static const char StubNativeFunctionName[] = "StubNativeFunction";
+
+
+void StubNativeFunction(Dart_NativeArguments arguments) {
+  // This is a stub function for the resolver
+  UNREACHABLE();
+}
+
+
+static Dart_NativeFunction StubNativeLookup(Dart_Handle name,
+                                            int argument_count,
+                                            bool* auto_setup_scope) {
+  return &StubNativeFunction;
+}
+
+
+static const uint8_t* StubNativeSymbol(Dart_NativeFunction nf) {
+  return reinterpret_cast<const uint8_t *>(StubNativeFunctionName);
+}
+
+
+static void SetupStubNativeResolver(size_t lib_index,
+                                    const Dart_QualifiedFunctionName* entry) {
+  // TODO(24686): Remove this.
+  Dart_Handle library_string = Dart_NewStringFromCString(entry->library_uri);
+  DART_CHECK_VALID(library_string);
+  Dart_Handle library = Dart_LookupLibrary(library_string);
+  // Embedder entry points may be setup in libraries that have not been
+  // explicitly loaded by the application script. In such cases, library lookup
+  // will fail. Manually load those libraries.
+  if (Dart_IsError(library)) {
+    static const uint32_t kLoadBufferMaxSize = 128;
+    char* load_buffer =
+        reinterpret_cast<char*>(calloc(kLoadBufferMaxSize, sizeof(char)));
+    snprintf(load_buffer,
+             kLoadBufferMaxSize,
+             "import '%s';",
+             DartUtils::GetStringValue(library_string));
+    Dart_Handle script_handle = Dart_NewStringFromCString(load_buffer);
+    memset(load_buffer, 0, kLoadBufferMaxSize);
+    snprintf(load_buffer,
+             kLoadBufferMaxSize,
+             "dart:_snapshot_%zu",
+             lib_index);
+    Dart_Handle script_url = Dart_NewStringFromCString(load_buffer);
+    free(load_buffer);
+    Dart_Handle loaded = Dart_LoadLibrary(script_url, script_handle, 0, 0);
+    DART_CHECK_VALID(loaded);
+
+    // Do a fresh lookup
+    library = Dart_LookupLibrary(library_string);
+  }
+
+  DART_CHECK_VALID(library);
+  Dart_Handle result =  Dart_SetNativeResolver(library,
+                                               &StubNativeLookup,
+                                               &StubNativeSymbol);
+  DART_CHECK_VALID(result);
+}
+
+
+static void ImportNativeEntryPointLibrariesIntoRoot(
+    const Dart_QualifiedFunctionName* entries) {
+  if (entries == NULL) {
+    return;
+  }
+
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == NULL) {
+      // The termination sentinel has null members.
+      break;
+    }
+    Dart_Handle entry_library =
+        Dart_LookupLibrary(Dart_NewStringFromCString(entry.library_uri));
+    DART_CHECK_VALID(entry_library);
+    Dart_Handle import_result = Dart_LibraryImportLibrary(
+        entry_library, Dart_RootLibrary(), Dart_EmptyString());
+    DART_CHECK_VALID(import_result);
+  }
+}
+
+
+static void SetupStubNativeResolversForPrecompilation(
+    const Dart_QualifiedFunctionName* entries) {
+
+  if (entries == NULL) {
+    return;
+  }
+
+  // Setup native resolvers for all libraries found in the manifest.
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == NULL) {
+      // The termination sentinel has null members.
+      break;
+    }
+    // Setup stub resolvers on loaded libraries
+    SetupStubNativeResolver(index, &entry);
+  }
+}
+
+
+static void CleanupEntryPointItem(const Dart_QualifiedFunctionName *entry) {
+  if (entry == NULL) {
+    return;
+  }
+  // The allocation used for these entries is zero'ed. So even in error cases,
+  // references to some entries will be null. Calling this on an already cleaned
+  // up entry is programmer error.
+  free(const_cast<char*>(entry->library_uri));
+  free(const_cast<char*>(entry->class_name));
+  free(const_cast<char*>(entry->function_name));
+}
+
+
+static void CleanupEntryPointsCollection(Dart_QualifiedFunctionName* entries) {
+  if (entries == NULL) {
+    return;
+  }
+
+  size_t index = 0;
+  while (true) {
+    Dart_QualifiedFunctionName entry = entries[index++];
+    if (entry.library_uri == NULL) {
+      break;
+    }
+    CleanupEntryPointItem(&entry);
+  }
+  free(entries);
+}
+
+
+char* ParserErrorStringCreate(const char* format, ...) {
+  static const size_t kErrorBufferSize = 256;
+
+  char* error_buffer =
+      reinterpret_cast<char*>(calloc(kErrorBufferSize, sizeof(char)));
+  va_list args;
+  va_start(args, format);
+  vsnprintf(error_buffer, kErrorBufferSize, format, args);
+  va_end(args);
+
+  // In case of error, the buffer is released by the caller
+  return error_buffer;
+}
+
+
+const char* ParseEntryNameForIndex(uint8_t index) {
+  switch (index) {
+    case 0:
+      return "Library";
+    case 1:
+      return "Class";
+    case 2:
+      return "Function";
+    default:
+      return "Unknown";
+  }
+  return NULL;
+}
+
+
+static bool ParseEntryPointsManifestSingleLine(
+    const char* line, Dart_QualifiedFunctionName* entry, char** error) {
+  bool success = true;
+  size_t offset = 0;
+  for (uint8_t i = 0; i < 3; i++) {
+    const char* component = strchr(line + offset, i == 2 ? '\n' : ',');
+    if (component == NULL) {
+      success = false;
+      *error = ParserErrorStringCreate(
+          "Manifest entries must be comma separated and newline terminated. "
+          "Could not parse '%s' on line '%s'",
+          ParseEntryNameForIndex(i), line);
+      break;
+    }
+
+    int64_t chars_read = component - (line + offset);
+    if (chars_read <= 0) {
+      success = false;
+      *error =
+          ParserErrorStringCreate("There is no '%s' specified on line '%s'",
+                                  ParseEntryNameForIndex(i), line);
+      break;
+    }
+
+    if (entry != NULL) {
+      // These allocations are collected in |CleanupEntryPointsCollection|.
+      char* entry_item =
+          reinterpret_cast<char*>(calloc(chars_read + 1, sizeof(char)));
+      memcpy(entry_item, line + offset, chars_read);
+
+      switch (i) {
+        case 0:  // library
+          entry->library_uri = entry_item;
+          break;
+        case 1:  // class
+          entry->class_name = entry_item;
+          break;
+        case 2:  // function
+          entry->function_name = entry_item;
+          break;
+        default:
+          free(entry_item);
+          success = false;
+          *error = ParserErrorStringCreate("Internal parser error\n");
+          break;
+      }
+    }
+
+    offset += chars_read + 1;
+  }
+  return success;
+}
+
+
+int64_t ParseEntryPointsManifestLines(FILE* file,
+                                      Dart_QualifiedFunctionName* collection) {
+  int64_t entries = 0;
+
+  static const int kManifestMaxLineLength = 1024;
+  char* line = reinterpret_cast<char*>(malloc(kManifestMaxLineLength));
+  size_t line_number = 0;
+  while (true) {
+    line_number++;
+    char* read_line = fgets(line, kManifestMaxLineLength, file);
+
+    if (read_line == NULL) {
+      if ((feof(file) != 0) && (ferror(file) != 0)) {
+        Log::PrintErr(
+            "Error while reading line number %zu. The manifest must be "
+            "terminated by a newline\n",
+            line_number);
+        entries = -1;
+      }
+      break;
+    }
+
+    Dart_QualifiedFunctionName* entry =
+        collection != NULL ? collection + entries : NULL;
+
+    char* error_buffer = NULL;
+    if (!ParseEntryPointsManifestSingleLine(read_line, entry, &error_buffer)) {
+      CleanupEntryPointItem(entry);
+      Log::PrintErr("Parser error on line %zu: %s\n", line_number,
+                    error_buffer);
+      free(error_buffer);
+      entries = -1;
+      break;
+    }
+
+    entries++;
+  }
+
+  free(line);
+
+  return entries;
+}
+
+
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestFile(
+    const char* path) {
+  if (path == NULL) {
+    return NULL;
+  }
+
+  FILE* file = fopen(path, "r");
+
+  if (file == NULL) {
+    Log::PrintErr("Could not open entry points manifest file\n");
+    return NULL;
+  }
+
+  // Parse the file once but don't store the results. This is done to first
+  // determine the number of entries in the manifest
+  int64_t entry_count = ParseEntryPointsManifestLines(file, NULL);
+
+  if (entry_count <= 0) {
+    Log::PrintErr(
+        "Manifest file specified is invalid or contained no entries\n");
+    fclose(file);
+    return NULL;
+  }
+
+  rewind(file);
+
+  // Allocate enough storage for the entries in the file plus a termination
+  // sentinel and parse it again to populate the allocation
+  Dart_QualifiedFunctionName* entries =
+      reinterpret_cast<Dart_QualifiedFunctionName*>(
+          calloc(entry_count + 1, sizeof(Dart_QualifiedFunctionName)));
+
+  int64_t parsed_entry_count = ParseEntryPointsManifestLines(file, entries);
+  ASSERT(parsed_entry_count == entry_count);
+
+  fclose(file);
+
+  // The entries allocation must be explicitly cleaned up via
+  // |CleanupEntryPointsCollection|
+  return entries;
+}
+
+
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestIfPresent() {
+  Dart_QualifiedFunctionName* entries =
+      ParseEntryPointsManifestFile(embedder_entry_points_manifest);
+  if (entries == NULL && IsSnapshottingForPrecompilation()) {
+    Log::PrintErr(
+        "Could not find native embedder entry points during precompilation\n");
+    exit(255);
+  }
+  return entries;
+}
+
+
 static void CreateAndWriteSnapshot() {
+  ASSERT(!IsSnapshottingForPrecompilation());
   Dart_Handle result;
   uint8_t* vm_isolate_buffer = NULL;
   intptr_t vm_isolate_size = 0;
   uint8_t* isolate_buffer = NULL;
   intptr_t isolate_size = 0;
 
   // First create a snapshot.
   result = Dart_CreateSnapshot(&vm_isolate_buffer,
                                &vm_isolate_size,
                                &isolate_buffer,
                                &isolate_size);
   CHECK_RESULT(result);
 
   // Now write the vm isolate and isolate snapshots out to the
   // specified file and exit.
   WriteSnapshotFile(vm_isolate_snapshot_filename,
                     vm_isolate_buffer,
                     vm_isolate_size);
   WriteSnapshotFile(isolate_snapshot_filename,
                     isolate_buffer,
                     isolate_size);
   Dart_ExitScope();
 
   // Shutdown the isolate.
   Dart_ShutdownIsolate();
 }
 
 
+static void CreateAndWritePrecompiledSnapshot(
+    Dart_QualifiedFunctionName* standalone_entry_points) {
+  ASSERT(IsSnapshottingForPrecompilation());
+  Dart_Handle result;
+  uint8_t* vm_isolate_buffer = NULL;
+  intptr_t vm_isolate_size = 0;
+  uint8_t* isolate_buffer = NULL;
+  intptr_t isolate_size = 0;
+  uint8_t* instructions_buffer = NULL;
+  intptr_t instructions_size = 0;
+
+  // Precompile with specified embedder entry points
+  result = Dart_Precompile(standalone_entry_points, true);
+  CHECK_RESULT(result);
+
+  // Create a precompiled snapshot. This gives us an instruction buffer with
+  // machine code
+  result = Dart_CreatePrecompiledSnapshot(&vm_isolate_buffer,
+                                          &vm_isolate_size,
+                                          &isolate_buffer,
+                                          &isolate_size,
+                                          &instructions_buffer,
+                                          &instructions_size);
+  CHECK_RESULT(result);
+
+  // Now write the vm isolate, isolate and instructions snapshots out to the
+  // specified files and exit.
+  WriteSnapshotFile(vm_isolate_snapshot_filename,
+                    vm_isolate_buffer,
+                    vm_isolate_size);
+  WriteSnapshotFile(isolate_snapshot_filename,
+                    isolate_buffer,
+                    isolate_size);
+  WriteSnapshotFile(instructions_snapshot_filename,
+                    instructions_buffer,
+                    instructions_size);
+  Dart_ExitScope();
+
+  // Shutdown the isolate.
+  Dart_ShutdownIsolate();
+}
+
+
 static void SetupForUriResolution() {
   // Set up the library tag handler for this isolate.
   Dart_Handle result = Dart_SetLibraryTagHandler(DartUtils::LibraryTagHandler);
   if (Dart_IsError(result)) {
     Log::PrintErr("%s", Dart_GetError(result));
     Dart_ExitScope();
     Dart_ShutdownIsolate();
     exit(255);
   }
   // This is a generic dart snapshot which needs builtin library setup.
@@ skipping 84 matching lines @@
 
   Thread::InitOnce();
   DartUtils::SetOriginalWorkingDirectory();
   // Start event handler.
   EventHandler::Start();
 
   vm_options.AddArgument("--load_deferred_eagerly");
   // Workaround until issue 21620 is fixed.
   // (https://github.com/dart-lang/sdk/issues/21620)
   vm_options.AddArgument("--no-concurrent_sweep");
+
+  if (IsSnapshottingForPrecompilation()) {
+    vm_options.AddArgument("--precompilation");
+  }
+
   Dart_SetVMFlags(vm_options.count(), vm_options.arguments());
 
   // Initialize the Dart VM.
   // Note: We don't expect isolates to be created from dart code during
   // snapshot generation.
   char* error = Dart_Initialize(
       NULL,
       NULL,
       CreateServiceIsolate,
       NULL,
@@ skipping 63 matching lines @@
       fprintf(stderr, "%s", error);
       free(error);
       exit(255);
     }
     Dart_EnterScope();
 
     // Set up the library tag handler in such a manner that it will use the
     // URL mapping specified on the command line to load the libraries.
     result = Dart_SetLibraryTagHandler(CreateSnapshotLibraryTagHandler);
     CHECK_RESULT(result);
+
+    Dart_QualifiedFunctionName* entry_points =
+        ParseEntryPointsManifestIfPresent();
+
+    SetupStubNativeResolversForPrecompilation(entry_points);
+
     // Load the specified script.
     library = LoadSnapshotCreationScript(app_script_name);
     VerifyLoaded(library);
+
+    ImportNativeEntryPointLibrariesIntoRoot(entry_points);
+
     // Ensure that we mark all libraries as loaded.
     result = Dart_FinalizeLoading(false);
     CHECK_RESULT(result);
-    CreateAndWriteSnapshot();
+
+    if (entry_points == NULL) {
+      ASSERT(!IsSnapshottingForPrecompilation());
+      CreateAndWriteSnapshot();
+    } else {
+      CreateAndWritePrecompiledSnapshot(entry_points);
+    }
+
+    CleanupEntryPointsCollection(entry_points);
 
     Dart_EnterIsolate(UriResolverIsolateScope::isolate);
     Dart_ShutdownIsolate();
   } else {
     SetupForGenericSnapshotCreation();
     CreateAndWriteSnapshot();
   }
   EventHandler::Stop();
   return 0;
 }
 
 }  // namespace bin
 }  // namespace dart
 
 int main(int argc, char** argv) {
   return dart::bin::main(argc, argv);
 }