Index: runtime/bin/gen_snapshot.cc |
diff --git a/runtime/bin/gen_snapshot.cc b/runtime/bin/gen_snapshot.cc |
index bb6d4f8502e425ff0748b4f84f39ead591311132..70356b5f365c763b6e62adc5fbf3bc154368d54d 100644 |
--- a/runtime/bin/gen_snapshot.cc |
+++ b/runtime/bin/gen_snapshot.cc |
@@ -9,6 +9,8 @@ |
#include <string.h> |
#include <stdio.h> |
+#include <cstdarg> |
+ |
#include "include/dart_api.h" |
#include "bin/builtin.h" |
@@ -38,6 +40,8 @@ namespace bin { |
// if so which file to write the snapshot into. |
static const char* vm_isolate_snapshot_filename = NULL; |
static const char* isolate_snapshot_filename = NULL; |
+static const char* instructions_snapshot_filename = NULL; |
+static const char* embedder_entry_points_manifest = NULL; |
static const char* package_root = NULL; |
@@ -88,6 +92,26 @@ static bool ProcessIsolateSnapshotOption(const char* option) { |
} |
+static bool ProcessInstructionsSnapshotOption(const char* option) { |
+ const char* name = ProcessOption(option, "--instructions_snapshot="); |
+ if (name != NULL) { |
+ instructions_snapshot_filename = name; |
+ return true; |
+ } |
+ return false; |
+} |
+ |
+ |
+static bool ProcessEmbedderEntryPointsManifestOption(const char* option) { |
+ const char* name = ProcessOption(option, "--embedder_entry_points_manifest="); |
+ if (name != NULL) { |
+ embedder_entry_points_manifest = name; |
+ return true; |
+ } |
+ return false; |
+} |
+ |
+ |
static bool ProcessPackageRootOption(const char* option) { |
const char* name = ProcessOption(option, "--package_root="); |
if (name != NULL) { |
@@ -127,6 +151,8 @@ static int ParseArguments(int argc, |
while ((i < argc) && IsValidFlag(argv[i], kPrefix, kPrefixLen)) { |
if (ProcessVmIsolateSnapshotOption(argv[i]) || |
ProcessIsolateSnapshotOption(argv[i]) || |
+ ProcessInstructionsSnapshotOption(argv[i]) || |
+ ProcessEmbedderEntryPointsManifestOption(argv[i]) || |
ProcessURLmappingOption(argv[i]) || |
ProcessPackageRootOption(argv[i])) { |
i += 1; |
@@ -154,10 +180,32 @@ static int ParseArguments(int argc, |
return -1; |
} |
+ if (instructions_snapshot_filename != NULL && |
+ embedder_entry_points_manifest == NULL) { |
+ Log::PrintErr( |
+ "Specifying an instructions snapshot filename indicates precompilation" |
+ ". But no embedder entry points manifest was specified.\n\n"); |
+ return -1; |
+ } |
+ |
+ if (embedder_entry_points_manifest != NULL && |
+ instructions_snapshot_filename == NULL) { |
+ Log::PrintErr( |
+ "Specifying the embedder entry points manifest indicates " |
+ "precompilation. But no instuctions snapshot was specified.\n\n"); |
+ return -1; |
+ } |
+ |
return 0; |
} |
+static bool IsSnapshottingForPrecompilation(void) { |
+ return embedder_entry_points_manifest != NULL && |
+ instructions_snapshot_filename != NULL; |
+} |
+ |
+ |
static void WriteSnapshotFile(const char* filename, |
const uint8_t* buffer, |
const intptr_t size) { |
@@ -420,27 +468,54 @@ static Dart_Handle LoadGenericSnapshotCreationScript( |
static void PrintUsage() { |
- Log::PrintErr( |
-"Usage:\n" |
-"\n" |
-" gen_snapshot [<vm-flags>] [<options>] \\\n" |
-" --snapshot=<out-file> [<dart-script-file>]\n" |
-"\n" |
-" Writes a snapshot of <dart-script-file> to <out-file>. If no\n" |
-" <dart-script-file> is passed, a generic snapshot of all the corelibs is\n" |
-" created. It is required to specify an output file name:\n" |
-"\n" |
-" --snapshot=<file> Generates a complete snapshot. Uses the url\n" |
-" mapping specified on the command line to load\n" |
-" the libraries.\n" |
-"Supported options:\n" |
-"\n" |
-"--package_root=<path>\n" |
-" Where to find packages, that is, \"package:...\" imports.\n" |
-"\n" |
-"--url_mapping=<mapping>\n" |
-" Uses the URL mapping(s) specified on the command line to load the\n" |
-" libraries. For use only with --snapshot=.\n"); |
+#define STRINGERIZE(...) #__VA_ARGS__ |
+ // clang-format off |
+ Log::PrintErr(STRINGERIZE( |
+Usage: \n |
+ gen_snapshot [<vm-flags>] [<options>] [<dart-script-file>] \n |
+ \n |
+ Writes a snapshot of <dart-script-file> to the specified snapshot files. If \n |
+ no <dart-script-file> is passed, a generic snapshot of all the corelibs is \n |
+ created. It is required to specify the VM isolate snapshot and the isolate \n |
+ snapshot. The other flags are related to precompilation and are optional. \n |
+ \n |
+ Precompilation: \n |
+ In order to configure the snapshotter for precompilation, both the \n |
+ instructions snapshot and embedder entry points manifest must be specified. \n |
+ Machine code for the target architecture will be dumped into the \n |
+ instructions snapshot. This must be linked into the target binary in a \n |
+ separate step. The embedder entry points manifest lists the standalone entry\n |
+ points into the VM. Not specifying these will cause the tree shaker to \n |
+ disregard the same as being used. The format of this manifest is as follows.\n |
+ Each line in the manifest is a comma separated list of three elements. The \n |
+ first entry is the library URI, the second entry is the class name and the \n |
+ final entry the function name. The file must be terminated with a newline \n |
+ charater. \n |
+ \n |
+ Example: \n |
+ dart:something,SomeClass,doSomething \n |
+ \n |
+ Supported options: \n |
+ --vm_isolate_snapshot=<file> A full snapshot is a compact \n |
+ --isolate_snapshot=<file> representation of the dart vm isolate \n |
+ heap and dart isolate heap states. \n |
+ Both these options are required \n |
+ \n |
+ --package_root=<path> Where to find packages, that is, \n |
+ package:... imports. \n |
+ \n |
+ --url_mapping=<mapping> Uses the URL mapping(s) specified on the\n |
+ command line to load the libraries. \n |
+ \n |
+ --instructions_snapshot=<file> (Precompilation only) Contains the \n |
+ assembly that must be linked into \n |
+ the target binary \n |
+ \n |
+ --embedder_entry_points_manifest=<file> (Precompilation only) Contains the\n |
+ stanalone embedder entry points \n |
+ )); |
+ // clang-format on |
+#undef STRINGERIZE |
} |
@@ -456,7 +531,326 @@ static void VerifyLoaded(Dart_Handle library) { |
} |
+static const char StubNativeFunctionName[] = "StubNativeFunction"; |
+ |
+ |
+void StubNativeFunction(Dart_NativeArguments arguments) { |
+ // This is a stub function for the resolver |
+ UNREACHABLE(); |
+} |
+ |
+ |
+static Dart_NativeFunction StubNativeLookup(Dart_Handle name, |
+ int argument_count, |
+ bool* auto_setup_scope) { |
+ return &StubNativeFunction; |
+} |
+ |
+ |
+static const uint8_t* StubNativeSymbol(Dart_NativeFunction nf) { |
+ return reinterpret_cast<const uint8_t *>(StubNativeFunctionName); |
+} |
+ |
+ |
+static void SetupStubNativeResolver(size_t lib_index, |
+ const Dart_QualifiedFunctionName* entry) { |
+ // TODO(24686): Remove this. |
+ Dart_Handle library_string = Dart_NewStringFromCString(entry->library_uri); |
+ DART_CHECK_VALID(library_string); |
+ Dart_Handle library = Dart_LookupLibrary(library_string); |
+ // Embedder entry points may be setup in libraries that have not been |
+ // explicitly loaded by the application script. In such cases, library lookup |
+ // will fail. Manually load those libraries. |
+ if (Dart_IsError(library)) { |
+ static const uint32_t kLoadBufferMaxSize = 128; |
+ char* load_buffer = |
+ reinterpret_cast<char*>(calloc(kLoadBufferMaxSize, sizeof(char))); |
+ snprintf(load_buffer, |
+ kLoadBufferMaxSize, |
+ "import '%s';", |
+ DartUtils::GetStringValue(library_string)); |
+ Dart_Handle script_handle = Dart_NewStringFromCString(load_buffer); |
+ memset(load_buffer, 0, kLoadBufferMaxSize); |
+ snprintf(load_buffer, |
+ kLoadBufferMaxSize, |
+ "dart:_snapshot_%zu", |
+ lib_index); |
+ Dart_Handle script_url = Dart_NewStringFromCString(load_buffer); |
+ free(load_buffer); |
+ Dart_Handle loaded = Dart_LoadLibrary(script_url, script_handle, 0, 0); |
+ DART_CHECK_VALID(loaded); |
+ |
+ // Do a fresh lookup |
+ library = Dart_LookupLibrary(library_string); |
+ } |
+ |
+ DART_CHECK_VALID(library); |
+ Dart_Handle result = Dart_SetNativeResolver(library, |
+ &StubNativeLookup, |
+ &StubNativeSymbol); |
+ DART_CHECK_VALID(result); |
+} |
+ |
+ |
+static void ImportNativeEntryPointLibrariesIntoRoot( |
+ const Dart_QualifiedFunctionName* entries) { |
+ if (entries == NULL) { |
+ return; |
+ } |
+ |
+ size_t index = 0; |
+ while (true) { |
+ Dart_QualifiedFunctionName entry = entries[index++]; |
+ if (entry.library_uri == NULL) { |
+ // The termination sentinel has null members. |
+ break; |
+ } |
+ Dart_Handle entry_library = |
+ Dart_LookupLibrary(Dart_NewStringFromCString(entry.library_uri)); |
+ DART_CHECK_VALID(entry_library); |
+ Dart_Handle import_result = Dart_LibraryImportLibrary( |
+ entry_library, Dart_RootLibrary(), Dart_EmptyString()); |
+ DART_CHECK_VALID(import_result); |
+ } |
+} |
+ |
+ |
+static void SetupStubNativeResolversForPrecompilation( |
+ const Dart_QualifiedFunctionName* entries) { |
+ |
+ if (entries == NULL) { |
+ return; |
+ } |
+ |
+ // Setup native resolvers for all libraries found in the manifest. |
+ size_t index = 0; |
+ while (true) { |
+ Dart_QualifiedFunctionName entry = entries[index++]; |
+ if (entry.library_uri == NULL) { |
+ // The termination sentinel has null members. |
+ break; |
+ } |
+ // Setup stub resolvers on loaded libraries |
+ SetupStubNativeResolver(index, &entry); |
+ } |
+} |
+ |
+ |
+static void CleanupEntryPointItem(const Dart_QualifiedFunctionName *entry) { |
+ if (entry == NULL) { |
+ return; |
+ } |
+ // The allocation used for these entries is zero'ed. So even in error cases, |
+ // references to some entries will be null. Calling this on an already cleaned |
+ // up entry is programmer error. |
+ free(const_cast<char*>(entry->library_uri)); |
+ free(const_cast<char*>(entry->class_name)); |
+ free(const_cast<char*>(entry->function_name)); |
+} |
+ |
+ |
+static void CleanupEntryPointsCollection(Dart_QualifiedFunctionName* entries) { |
+ if (entries == NULL) { |
+ return; |
+ } |
+ |
+ size_t index = 0; |
+ while (true) { |
+ Dart_QualifiedFunctionName entry = entries[index++]; |
+ if (entry.library_uri == NULL) { |
+ break; |
+ } |
+ CleanupEntryPointItem(&entry); |
+ } |
+ free(entries); |
+} |
+ |
+ |
+char* ParserErrorStringCreate(const char* format, ...) { |
+ static const size_t kErrorBufferSize = 256; |
+ |
+ char* error_buffer = |
+ reinterpret_cast<char*>(calloc(kErrorBufferSize, sizeof(char))); |
+ va_list args; |
+ va_start(args, format); |
+ vsnprintf(error_buffer, kErrorBufferSize, format, args); |
+ va_end(args); |
+ |
+ // In case of error, the buffer is released by the caller |
+ return error_buffer; |
+} |
+ |
+ |
+const char* ParseEntryNameForIndex(uint8_t index) { |
+ switch (index) { |
+ case 0: |
+ return "Library"; |
+ case 1: |
+ return "Class"; |
+ case 2: |
+ return "Function"; |
+ default: |
+ return "Unknown"; |
+ } |
+ return NULL; |
+} |
+ |
+ |
+static bool ParseEntryPointsManifestSingleLine( |
+ const char* line, Dart_QualifiedFunctionName* entry, char** error) { |
+ bool success = true; |
+ size_t offset = 0; |
+ for (uint8_t i = 0; i < 3; i++) { |
+ const char* component = strchr(line + offset, i == 2 ? '\n' : ','); |
+ if (component == NULL) { |
+ success = false; |
+ *error = ParserErrorStringCreate( |
+ "Manifest entries must be comma separated and newline terminated. " |
+ "Could not parse '%s' on line '%s'", |
+ ParseEntryNameForIndex(i), line); |
+ break; |
+ } |
+ |
+ int64_t chars_read = component - (line + offset); |
+ if (chars_read <= 0) { |
+ success = false; |
+ *error = |
+ ParserErrorStringCreate("There is no '%s' specified on line '%s'", |
+ ParseEntryNameForIndex(i), line); |
+ break; |
+ } |
+ |
+ if (entry != NULL) { |
+ // These allocations are collected in |CleanupEntryPointsCollection|. |
+ char* entry_item = |
+ reinterpret_cast<char*>(calloc(chars_read + 1, sizeof(char))); |
+ memcpy(entry_item, line + offset, chars_read); |
+ |
+ switch (i) { |
+ case 0: // library |
+ entry->library_uri = entry_item; |
+ break; |
+ case 1: // class |
+ entry->class_name = entry_item; |
+ break; |
+ case 2: // function |
+ entry->function_name = entry_item; |
+ break; |
+ default: |
+ free(entry_item); |
+ success = false; |
+ *error = ParserErrorStringCreate("Internal parser error\n"); |
+ break; |
+ } |
+ } |
+ |
+ offset += chars_read + 1; |
+ } |
+ return success; |
+} |
+ |
+ |
+int64_t ParseEntryPointsManifestLines(FILE* file, |
+ Dart_QualifiedFunctionName* collection) { |
+ int64_t entries = 0; |
+ |
+ static const int kManifestMaxLineLength = 1024; |
+ char* line = reinterpret_cast<char*>(malloc(kManifestMaxLineLength)); |
+ size_t line_number = 0; |
+ while (true) { |
+ line_number++; |
+ char* read_line = fgets(line, kManifestMaxLineLength, file); |
+ |
+ if (read_line == NULL) { |
+ if ((feof(file) != 0) && (ferror(file) != 0)) { |
+ Log::PrintErr( |
+ "Error while reading line number %zu. The manifest must be " |
+ "terminated by a newline\n", |
+ line_number); |
+ entries = -1; |
+ } |
+ break; |
+ } |
+ |
+ Dart_QualifiedFunctionName* entry = |
+ collection != NULL ? collection + entries : NULL; |
+ |
+ char* error_buffer = NULL; |
+ if (!ParseEntryPointsManifestSingleLine(read_line, entry, &error_buffer)) { |
+ CleanupEntryPointItem(entry); |
+ Log::PrintErr("Parser error on line %zu: %s\n", line_number, |
+ error_buffer); |
+ free(error_buffer); |
+ entries = -1; |
+ break; |
+ } |
+ |
+ entries++; |
+ } |
+ |
+ free(line); |
+ |
+ return entries; |
+} |
+ |
+ |
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestFile( |
+ const char* path) { |
+ if (path == NULL) { |
+ return NULL; |
+ } |
+ |
+ FILE* file = fopen(path, "r"); |
+ |
+ if (file == NULL) { |
+ Log::PrintErr("Could not open entry points manifest file\n"); |
+ return NULL; |
+ } |
+ |
+ // Parse the file once but don't store the results. This is done to first |
+ // determine the number of entries in the manifest |
+ int64_t entry_count = ParseEntryPointsManifestLines(file, NULL); |
+ |
+ if (entry_count <= 0) { |
+ Log::PrintErr( |
+ "Manifest file specified is invalid or contained no entries\n"); |
+ fclose(file); |
+ return NULL; |
+ } |
+ |
+ rewind(file); |
+ |
+ // Allocate enough storage for the entries in the file plus a termination |
+ // sentinel and parse it again to populate the allocation |
+ Dart_QualifiedFunctionName* entries = |
+ reinterpret_cast<Dart_QualifiedFunctionName*>( |
+ calloc(entry_count + 1, sizeof(Dart_QualifiedFunctionName))); |
+ |
+ int64_t parsed_entry_count = ParseEntryPointsManifestLines(file, entries); |
+ ASSERT(parsed_entry_count == entry_count); |
+ |
+ fclose(file); |
+ |
+ // The entries allocation must be explicitly cleaned up via |
+ // |CleanupEntryPointsCollection| |
+ return entries; |
+} |
+ |
+ |
+static Dart_QualifiedFunctionName* ParseEntryPointsManifestIfPresent() { |
+ Dart_QualifiedFunctionName* entries = |
+ ParseEntryPointsManifestFile(embedder_entry_points_manifest); |
+ if (entries == NULL && IsSnapshottingForPrecompilation()) { |
+ Log::PrintErr( |
+ "Could not find native embedder entry points during precompilation\n"); |
+ exit(255); |
+ } |
+ return entries; |
+} |
+ |
+ |
static void CreateAndWriteSnapshot() { |
+ ASSERT(!IsSnapshottingForPrecompilation()); |
Dart_Handle result; |
uint8_t* vm_isolate_buffer = NULL; |
intptr_t vm_isolate_size = 0; |
@@ -485,6 +879,49 @@ static void CreateAndWriteSnapshot() { |
} |
+static void CreateAndWritePrecompiledSnapshot( |
+ Dart_QualifiedFunctionName* standalone_entry_points) { |
+ ASSERT(IsSnapshottingForPrecompilation()); |
+ Dart_Handle result; |
+ uint8_t* vm_isolate_buffer = NULL; |
+ intptr_t vm_isolate_size = 0; |
+ uint8_t* isolate_buffer = NULL; |
+ intptr_t isolate_size = 0; |
+ uint8_t* instructions_buffer = NULL; |
+ intptr_t instructions_size = 0; |
+ |
+ // Precompile with specified embedder entry points |
+ result = Dart_Precompile(standalone_entry_points, true); |
+ CHECK_RESULT(result); |
+ |
+ // Create a precompiled snapshot. This gives us an instruction buffer with |
+ // machine code |
+ result = Dart_CreatePrecompiledSnapshot(&vm_isolate_buffer, |
+ &vm_isolate_size, |
+ &isolate_buffer, |
+ &isolate_size, |
+ &instructions_buffer, |
+ &instructions_size); |
+ CHECK_RESULT(result); |
+ |
+ // Now write the vm isolate, isolate and instructions snapshots out to the |
+ // specified files and exit. |
+ WriteSnapshotFile(vm_isolate_snapshot_filename, |
+ vm_isolate_buffer, |
+ vm_isolate_size); |
+ WriteSnapshotFile(isolate_snapshot_filename, |
+ isolate_buffer, |
+ isolate_size); |
+ WriteSnapshotFile(instructions_snapshot_filename, |
+ instructions_buffer, |
+ instructions_size); |
+ Dart_ExitScope(); |
+ |
+ // Shutdown the isolate. |
+ Dart_ShutdownIsolate(); |
+} |
+ |
+ |
static void SetupForUriResolution() { |
// Set up the library tag handler for this isolate. |
Dart_Handle result = Dart_SetLibraryTagHandler(DartUtils::LibraryTagHandler); |
@@ -589,6 +1026,11 @@ int main(int argc, char** argv) { |
// Workaround until issue 21620 is fixed. |
// (https://github.com/dart-lang/sdk/issues/21620) |
vm_options.AddArgument("--no-concurrent_sweep"); |
+ |
+ if (IsSnapshottingForPrecompilation()) { |
+ vm_options.AddArgument("--precompilation"); |
+ } |
+ |
Dart_SetVMFlags(vm_options.count(), vm_options.arguments()); |
// Initialize the Dart VM. |
@@ -672,13 +1114,30 @@ int main(int argc, char** argv) { |
// URL mapping specified on the command line to load the libraries. |
result = Dart_SetLibraryTagHandler(CreateSnapshotLibraryTagHandler); |
CHECK_RESULT(result); |
+ |
+ Dart_QualifiedFunctionName* entry_points = |
+ ParseEntryPointsManifestIfPresent(); |
+ |
+ SetupStubNativeResolversForPrecompilation(entry_points); |
+ |
// Load the specified script. |
library = LoadSnapshotCreationScript(app_script_name); |
VerifyLoaded(library); |
+ |
+ ImportNativeEntryPointLibrariesIntoRoot(entry_points); |
+ |
// Ensure that we mark all libraries as loaded. |
result = Dart_FinalizeLoading(false); |
CHECK_RESULT(result); |
- CreateAndWriteSnapshot(); |
+ |
+ if (entry_points == NULL) { |
+ ASSERT(!IsSnapshottingForPrecompilation()); |
+ CreateAndWriteSnapshot(); |
+ } else { |
+ CreateAndWritePrecompiledSnapshot(entry_points); |
+ } |
+ |
+ CleanupEntryPointsCollection(entry_points); |
Dart_EnterIsolate(UriResolverIsolateScope::isolate); |
Dart_ShutdownIsolate(); |