Use a single file for app snapshots.

runtime/bin/main.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
 
 #include "include/dart_api.h"
 #include "include/dart_tools_api.h"
@@ skipping 79 matching lines @@
 // the same way as precompilation before main, then continue running in the
 // same process.
 // Always set this with dart_noopt.
 #if defined(DART_PRECOMPILER) && !defined(DART_NO_SNAPSHOT)
 static const bool is_noopt = true;
 #else
 static const bool is_noopt = false;
 #endif
 
 
-extern const char* kPrecompiledLibraryName;
+extern const char* kPrecompiledVMIsolateSymbolName;
+extern const char* kPrecompiledIsolateSymbolName;
 extern const char* kPrecompiledInstructionsSymbolName;
 extern const char* kPrecompiledDataSymbolName;
 
-static const char* kVMIsolateSuffix = "snapshot.vmisolate";
-static const char* kIsolateSuffix = "snapshot.isolate";
-static const char* kAssemblySuffix = "snapshot.S";
-static const char* kInstructionsSuffix = "snapshot.instructions";
-static const char* kRODataSuffix = "snapshot.rodata";
-
 
 // Global flag that is used to indicate that we want to trace resolution of
 // URIs and the loading of libraries, parts and scripts.
 static bool trace_loading = false;
 
 
 static const char* DEFAULT_VM_SERVICE_SERVER_IP = "127.0.0.1";
 static const int DEFAULT_VM_SERVICE_SERVER_PORT = 8181;
 // VM Service options.
 static const char* vm_service_server_ip = DEFAULT_VM_SERVICE_SERVER_IP;
@@ skipping 1080 matching lines @@
               "Unable to write file %s for writing snapshot\n",
               qualified_filename);
   }
   file->Release();
   if (concat != NULL) {
     delete concat;
   }
 }
 
 
-static void ReadSnapshotFile(const char* snapshot_directory,
-                             const char* filename,
-                             const uint8_t** buffer) {
-  char* concat = NULL;
-  const char* qualified_filename;
-  if ((snapshot_directory != NULL) && (strlen(snapshot_directory) > 0)) {
-    intptr_t len = snprintf(NULL, 0, "%s/%s", snapshot_directory, filename);
-    concat = new char[len + 1];
-    snprintf(concat, len + 1, "%s/%s", snapshot_directory, filename);
-    qualified_filename = concat;
+static const int64_t kAppSnapshotHeaderSize = 5 * sizeof(int64_t);  // NOLINT
+static const int64_t kAppSnapshotMagicNumber = 0xf6f6dcdc;
+static const int64_t kAppSnapshotPageSize = 4 * KB;
+
+
+static bool ReadAppSnapshotBlobs(const char* script_name,
+                                 const uint8_t** vmisolate_buffer,
+                                 const uint8_t** isolate_buffer,
+                                 const uint8_t** instructions_buffer,
+                                 const uint8_t** rodata_buffer) {
+  File* file = File::Open(script_name, File::kRead);
+  if (file == NULL) {
+    return false;
+  }
+  if (file->Length() < kAppSnapshotHeaderSize) {
+    file->Release();
+    return false;
+  }
+  int64_t magic = 0;
+  int64_t vmisolate_size = 0;
+  int64_t isolate_size = 0;
+  int64_t instructions_size = 0;
+  int64_t rodata_size = 0;
+  if (!file->ReadFully(&magic, sizeof(magic))) {
+    file->Release();
+    return false;
+  }
+  if (magic != kAppSnapshotMagicNumber) {
+    file->Release();
+    return false;
+  }
+  file->ReadFully(&vmisolate_size, sizeof(vmisolate_size));
+  file->ReadFully(&isolate_size, sizeof(isolate_size));
+  file->ReadFully(&instructions_size, sizeof(instructions_size));
+  file->ReadFully(&rodata_size, sizeof(rodata_size));
+  ASSERT(file->Position() == kAppSnapshotHeaderSize);

[siva, 2016/10/12 20:37:33]

Instead of these individual reads of the magic number and various sizes why not
do a single read of kAppSnapshotHeaderSize into a buffer that could then be used
to compute the magic number and the various sizes.

[rmacnak, 2016/10/13 01:08:16]

Done.

+
+  int64_t position = file->Position();
+  position = Utils::RoundUp(position, kAppSnapshotPageSize);
+  *vmisolate_buffer = reinterpret_cast<const uint8_t*>(
+      file->MapReadOnly(position, vmisolate_size));
+  if (*vmisolate_buffer == NULL) {
+    ErrorExit(kErrorExitCode, "Failed to memory map snapshot\n");
+  }
+  position += vmisolate_size;
+
+  position = Utils::RoundUp(position, kAppSnapshotPageSize);
+  *isolate_buffer = reinterpret_cast<const uint8_t*>(
+      file->MapReadOnly(position, isolate_size));
+  if (*isolate_buffer == NULL) {
+    ErrorExit(kErrorExitCode, "Failed to memory map snapshot\n");
+  }
+  position += isolate_size;
+
+  if (instructions_size == 0) {
+    *instructions_buffer = NULL;
   } else {
-    qualified_filename = filename;
-  }
-
-  void* file = DartUtils::OpenFile(qualified_filename, false);
+    position = Utils::RoundUp(position, kAppSnapshotPageSize);
+    *instructions_buffer = reinterpret_cast<const uint8_t*>(
+        file->MapReadExecute(position, instructions_size));
+    if (*instructions_buffer == NULL) {
+      ErrorExit(kErrorExitCode, "Failed to memory map snapshot\n");
+    }
+    position += instructions_size;
+  }
+
+  if (rodata_size == 0) {
+    *rodata_buffer = NULL;
+  } else {
+    position = Utils::RoundUp(position, kAppSnapshotPageSize);
+    *rodata_buffer = reinterpret_cast<const uint8_t*>(
+        file->MapReadOnly(position, rodata_size));
+    if (*rodata_buffer == NULL) {
+      ErrorExit(kErrorExitCode, "Failed to memory map snapshot\n");
+    }
+  }
+

[siva, 2016/10/12 20:37:33]

Can the order of rodata be swapped with instructions then a single
mmap(PROT_READ) call could be done for the 3 read only sections and the pointers
vmisolate_buffer, isolate_buffer and rodata_buffer could be updated to point to
the addresses into the value returned by that mmap call.

[rmacnak, 2016/10/13 01:08:16]

Done.

+  return true;

[siva, 2016/10/12 20:37:33]

Missing file->Release() for the success case?
On Linux at least it is ok to class the fd after an mmap is done not sure if
there are issues on MacOS or Windows.

[rmacnak, 2016/10/13 01:08:16]

Done.

+}
+
+
+static bool ReadAppSnapshotDynamicLibrary(const char* script_name,
+                                          const uint8_t** vmisolate_buffer,
+                                          const uint8_t** isolate_buffer,
+                                          const uint8_t** instructions_buffer,
+                                          const uint8_t** rodata_buffer) {
+  void* library = Extensions::LoadExtensionLibrary(script_name);

[siva, 2016/10/12 20:37:33]

Can you make sure that this will work for any name on both linux/macos

[rmacnak, 2016/10/13 01:08:16]

Done, using the same for both platforms in the test harness now.

+  if (library == NULL) {
+    return false;
+  }
+
+  *vmisolate_buffer = reinterpret_cast<const uint8_t*>(
+      Extensions::ResolveSymbol(library, kPrecompiledVMIsolateSymbolName));
+  if (*vmisolate_buffer == NULL) {
+    ErrorExit(kErrorExitCode, "Failed to resolve symbol '%s'\n",
+              kPrecompiledVMIsolateSymbolName);
+  }
+
+  *isolate_buffer = reinterpret_cast<const uint8_t*>(
+      Extensions::ResolveSymbol(library, kPrecompiledIsolateSymbolName));
+  if (*isolate_buffer == NULL) {
+    ErrorExit(kErrorExitCode, "Failed to resolve symbol '%s'\n",
+              kPrecompiledIsolateSymbolName);
+  }
+
+  *instructions_buffer = reinterpret_cast<const uint8_t*>(
+      Extensions::ResolveSymbol(library, kPrecompiledInstructionsSymbolName));
+  if (*instructions_buffer == NULL) {
+    ErrorExit(kErrorExitCode, "Failed to resolve symbol '%s'\n",
+              kPrecompiledInstructionsSymbolName);
+  }
+
+  *rodata_buffer = reinterpret_cast<const uint8_t*>(
+      Extensions::ResolveSymbol(library, kPrecompiledDataSymbolName));
+  if (*rodata_buffer == NULL) {
+    ErrorExit(kErrorExitCode, "Failed to resolve symbol '%s'\n",
+              kPrecompiledDataSymbolName);
+  }
+
+  return true;
+}
+
+
+static bool ReadAppSnapshot(const char* script_name,
+                            const uint8_t** vmisolate_buffer,
+                            const uint8_t** isolate_buffer,
+                            const uint8_t** instructions_buffer,
+                            const uint8_t** rodata_buffer) {
+  if (ReadAppSnapshotBlobs(script_name,
+                           vmisolate_buffer,
+                           isolate_buffer,
+                           instructions_buffer,
+                           rodata_buffer)) {
+    return true;
+  }
+  return ReadAppSnapshotDynamicLibrary(script_name,
+                                       vmisolate_buffer,
+                                       isolate_buffer,
+                                       instructions_buffer,
+                                       rodata_buffer);
+}
+
+
+static bool WriteInt64(File* file, int64_t size) {
+  return file->WriteFully(&size, sizeof(size));
+}
+
+
+static void WriteAppSnapshot(const char* filename,
+                             uint8_t* vmisolate_buffer,
+                             intptr_t vmisolate_size,
+                             uint8_t* isolate_buffer,
+                             intptr_t isolate_size,
+                             uint8_t* instructions_buffer,
+                             intptr_t instructions_size,
+                             uint8_t* rodata_buffer,
+                             intptr_t rodata_size) {
+  File* file = File::Open(filename, File::kWriteTruncate);
   if (file == NULL) {
-    fprintf(stderr,
-            "Error: Unable to open file %s for reading snapshot\n",
-            qualified_filename);
-    fflush(stderr);
-    Platform::Exit(kErrorExitCode);
-  }
-  intptr_t len = -1;
-  DartUtils::ReadFile(buffer, &len, file);
-  if ((*buffer == NULL) || (len == -1)) {
-    fprintf(stderr,
-            "Error: Unable to read snapshot file %s\n", qualified_filename);
-    fflush(stderr);
-    Platform::Exit(kErrorExitCode);
-  }
-  DartUtils::CloseFile(file);
-  if (concat != NULL) {
-    delete[] concat;
-  }
-}
-
-
-static void ReadExecutableSnapshotFile(const char* snapshot_directory,
-                                       const char* filename,
-                                       const uint8_t** buffer) {
-  char* concat = NULL;
-  const char* qualified_filename;
-  if ((snapshot_directory != NULL) && (strlen(snapshot_directory) > 0)) {
-    intptr_t len = snprintf(NULL, 0, "%s/%s", snapshot_directory, filename);
-    concat = new char[len + 1];
-    snprintf(concat, len + 1, "%s/%s", snapshot_directory, filename);
-    qualified_filename = concat;
-  } else {
-    qualified_filename = filename;
-  }
-
-  intptr_t len = -1;
-  *buffer = reinterpret_cast<uint8_t*>(
-      DartUtils::MapExecutable(qualified_filename, &len));
-  if ((*buffer == NULL) || (len == -1)) {
-    fprintf(stderr,
-            "Error: Unable to read snapshot file %s\n", qualified_filename);
-    fflush(stderr);
-    Platform::Exit(kErrorExitCode);
-  }
-  if (concat != NULL) {
-    delete[] concat;
-  }
-}
-
-
-static void* LoadLibrarySymbol(const char* snapshot_directory,
-                               const char* libname,
-                               const char* symname) {
-  char* concat = NULL;
-  const char* qualified_libname;
-  if ((snapshot_directory != NULL) && (strlen(snapshot_directory) > 0)) {
-    intptr_t len = snprintf(NULL, 0, "%s/%s", snapshot_directory, libname);
-    concat = new char[len + 1];
-    snprintf(concat, len + 1, "%s/%s", snapshot_directory, libname);
-    qualified_libname = concat;
-  } else {
-    qualified_libname = libname;
-  }
-  void* library = Extensions::LoadExtensionLibrary(qualified_libname);
-  if (concat != NULL) {
-    delete concat;
-  }
-  if (library == NULL) {
-    return NULL;
-  }
-  return Extensions::ResolveSymbol(library, symname);
-}
-
-
+    ErrorExit(kErrorExitCode, "Unable to write snapshot file '%s'\n", filename);
+  }
+
+  file->WriteFully(&kAppSnapshotMagicNumber, sizeof(kAppSnapshotMagicNumber));
+  WriteInt64(file, vmisolate_size);
+  WriteInt64(file, isolate_size);
+  WriteInt64(file, instructions_size);
+  WriteInt64(file, rodata_size);
+  ASSERT(file->Position() == kAppSnapshotHeaderSize);
+
+  file->SetPosition(Utils::RoundUp(file->Position(), kAppSnapshotPageSize));
+  if (!file->WriteFully(vmisolate_buffer, vmisolate_size)) {
+    ErrorExit(kErrorExitCode, "Unable to write snapshot file '%s'\n", filename);
+  }
+
+  file->SetPosition(Utils::RoundUp(file->Position(), kAppSnapshotPageSize));
+  if (!file->WriteFully(isolate_buffer, isolate_size)) {
+    ErrorExit(kErrorExitCode, "Unable to write snapshot file '%s'\n", filename);
+  }
+
+  if (instructions_size != 0) {
+    file->SetPosition(Utils::RoundUp(file->Position(), kAppSnapshotPageSize));
+    if (!file->WriteFully(instructions_buffer, instructions_size)) {
+      ErrorExit(kErrorExitCode, "Unable to write snapshot file '%s'\n",
+                filename);
+    }
+  }
+
+  if (rodata_size != 0) {
+    file->SetPosition(Utils::RoundUp(file->Position(), kAppSnapshotPageSize));
+    if (!file->WriteFully(rodata_buffer, rodata_size)) {
+      ErrorExit(kErrorExitCode, "Unable to write snapshot file '%s'\n",
+                filename);
+    }
+  }
+
+  file->Flush();
+  file->Release();
+}
+
+
 static void GenerateScriptSnapshot() {
   // First create a snapshot.
   uint8_t* buffer = NULL;
   intptr_t size = 0;
   Dart_Handle result = Dart_CreateScriptSnapshot(&buffer, &size);
   if (Dart_IsError(result)) {
     ErrorExit(kErrorExitCode, "%s\n", Dart_GetError(result));
   }
 
   WriteSnapshotFile(NULL, snapshot_filename, true, buffer, size);
@@ skipping 17 matching lines @@
         &vm_isolate_buffer,
         &vm_isolate_size,
         &isolate_buffer,
         &isolate_size,
         &instructions_blob_buffer,
         &instructions_blob_size,
         &rodata_blob_buffer,
         &rodata_blob_size);
   } else {
     result = Dart_CreatePrecompiledSnapshotAssembly(
-        &vm_isolate_buffer,
-        &vm_isolate_size,
-        &isolate_buffer,
-        &isolate_size,
         &assembly_buffer,
         &assembly_size);
   }
   if (Dart_IsError(result)) {
     ErrorExit(kErrorExitCode, "%s\n", Dart_GetError(result));
   }
-  WriteSnapshotFile(snapshot_filename, kVMIsolateSuffix,
-                    false,
-                    vm_isolate_buffer,
-                    vm_isolate_size);
-  WriteSnapshotFile(snapshot_filename, kIsolateSuffix,
-                    false,
-                    isolate_buffer,
-                    isolate_size);
   if (use_blobs) {
-    WriteSnapshotFile(snapshot_filename, kInstructionsSuffix,
-                      false,
-                      instructions_blob_buffer,
-                      instructions_blob_size);
-    WriteSnapshotFile(snapshot_filename, kRODataSuffix,
-                      false,
-                      rodata_blob_buffer,
-                      rodata_blob_size);
+    WriteAppSnapshot(snapshot_filename,
+                     vm_isolate_buffer,
+                     vm_isolate_size,
+                     isolate_buffer,
+                     isolate_size,
+                     instructions_blob_buffer,
+                     instructions_blob_size,
+                     rodata_blob_buffer,
+                     rodata_blob_size);
   } else {
-    WriteSnapshotFile(snapshot_filename, kAssemblySuffix,
+    WriteSnapshotFile(NULL, snapshot_filename,
                       false,
                       assembly_buffer,
                       assembly_size);
   }
 }
 
 
 static void GeneratePrecompiledJITSnapshot() {
-  if (!use_blobs) {
-    ErrorExit(kErrorExitCode,
-              "Generating app JIT snapshots as assembly unimplemented\n");
-  }
   uint8_t* vm_isolate_buffer = NULL;
   intptr_t vm_isolate_size = 0;
   uint8_t* isolate_buffer = NULL;
   intptr_t isolate_size = 0;
   uint8_t* instructions_blob_buffer = NULL;
   intptr_t instructions_blob_size = 0;
   uint8_t* rodata_blob_buffer = NULL;
   intptr_t rodata_blob_size = 0;
   Dart_Handle result = Dart_CreateAppJITSnapshot(
       &vm_isolate_buffer,
       &vm_isolate_size,
       &isolate_buffer,
       &isolate_size,
       &instructions_blob_buffer,
       &instructions_blob_size,
       &rodata_blob_buffer,
       &rodata_blob_size);
   if (Dart_IsError(result)) {
     ErrorExit(kErrorExitCode, "%s\n", Dart_GetError(result));
   }
-  WriteSnapshotFile(snapshot_filename, kVMIsolateSuffix,
-                    false,
-                    vm_isolate_buffer,
-                    vm_isolate_size);
-  WriteSnapshotFile(snapshot_filename, kIsolateSuffix,
-                    false,
-                    isolate_buffer,
-                    isolate_size);
-  WriteSnapshotFile(snapshot_filename, kInstructionsSuffix,
-                    false,
-                    instructions_blob_buffer,
-                    instructions_blob_size);
-  WriteSnapshotFile(snapshot_filename, kRODataSuffix,
-                    false,
-                    rodata_blob_buffer,
-                    rodata_blob_size);
+  WriteAppSnapshot(snapshot_filename,
+                   vm_isolate_buffer,
+                   vm_isolate_size,
+                   isolate_buffer,
+                   isolate_size,
+                   instructions_blob_buffer,
+                   instructions_blob_size,
+                   rodata_blob_buffer,
+                   rodata_blob_size);
 }
 
 
 static void GenerateFullSnapshot() {
   // Create a full snapshot of the script.
   Dart_Handle result;
   uint8_t* vm_isolate_buffer = NULL;
   intptr_t vm_isolate_size = 0;
   uint8_t* isolate_buffer = NULL;
   intptr_t isolate_size = 0;
 
   result = Dart_CreateSnapshot(&vm_isolate_buffer,
                                &vm_isolate_size,
                                &isolate_buffer,
                                &isolate_size);
   if (Dart_IsError(result)) {
     ErrorExit(kErrorExitCode, "%s\n", Dart_GetError(result));
   }
 
-  WriteSnapshotFile(snapshot_filename,
-                    kVMIsolateSuffix,
-                    false,
-                    vm_isolate_buffer,
-                    vm_isolate_size);
-  WriteSnapshotFile(snapshot_filename,
-                    kIsolateSuffix,
-                    false,
-                    isolate_buffer,
-                    isolate_size);
+  WriteAppSnapshot(snapshot_filename,
+                   vm_isolate_buffer,
+                   vm_isolate_size,
+                   isolate_buffer,
+                   isolate_size,
+                   NULL, 0, NULL, 0);
 }
 
 
 #define CHECK_RESULT(result)                                                   \
   if (Dart_IsError(result)) {                                                  \
     if (Dart_IsVMRestartRequest(result)) {                                     \
       Dart_ExitScope();                                                        \
       Dart_ShutdownIsolate();                                                  \
       return true;                                                             \
     }                                                                          \
@@ skipping 318 matching lines @@
 
   Loader::InitOnce();
 
   if (!DartUtils::SetOriginalWorkingDirectory()) {
     OSError err;
     fprintf(stderr, "Error determining current directory: %s\n", err.message());
     fflush(stderr);
     Platform::Exit(kErrorExitCode);
   }
 
+  const uint8_t* instructions_snapshot = NULL;
+  const uint8_t* data_snapshot = NULL;
+
+  if (ReadAppSnapshot(script_name,
+                      &vm_isolate_snapshot_buffer,
+                      &isolate_snapshot_buffer,
+                      &instructions_snapshot,
+                      &data_snapshot)) {
+    run_app_snapshot = true;
+  }
+
 #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
   // Constant true if PRODUCT or DART_PRECOMPILED_RUNTIME.
   if ((gen_snapshot_kind != kNone) || run_app_snapshot) {
     vm_options.AddArgument("--load_deferred_eagerly");
   }
 #endif
 
   if (gen_snapshot_kind == kAppJITAfterRun) {
     vm_options.AddArgument("--fields_may_be_reset");
   }
   if ((gen_snapshot_kind == kAppAOT) || is_noopt) {
     vm_options.AddArgument("--precompilation");
   }
 #if defined(DART_PRECOMPILED_RUNTIME)
   vm_options.AddArgument("--precompilation");
 #endif
 
   Dart_SetVMFlags(vm_options.count(), vm_options.arguments());
 
   // Start event handler.
   TimerUtils::InitOnce();
   EventHandler::Start();
 
-  const uint8_t* instructions_snapshot = NULL;
-  const uint8_t* data_snapshot = NULL;
-  if (run_app_snapshot) {
-    ReadSnapshotFile(snapshot_filename, kVMIsolateSuffix,
-                     &vm_isolate_snapshot_buffer);
-    ReadSnapshotFile(snapshot_filename, kIsolateSuffix,
-                     &isolate_snapshot_buffer);
-    if (use_blobs) {
-      ReadExecutableSnapshotFile(snapshot_filename,
-                                 kInstructionsSuffix,
-                                 &instructions_snapshot);
-      ReadSnapshotFile(snapshot_filename, kRODataSuffix,
-                       &data_snapshot);
-    } else {
-      instructions_snapshot = reinterpret_cast<const uint8_t*>(
-          LoadLibrarySymbol(snapshot_filename,
-                            kPrecompiledLibraryName,
-                            kPrecompiledInstructionsSymbolName));
-      data_snapshot = reinterpret_cast<const uint8_t*>(
-          LoadLibrarySymbol(snapshot_filename,
-                            kPrecompiledLibraryName,
-                            kPrecompiledDataSymbolName));
-    }
-  }
-
   // Initialize the Dart VM.
   Dart_InitializeParams init_params;
   memset(&init_params, 0, sizeof(init_params));
   init_params.version = DART_INITIALIZE_PARAMS_CURRENT_VERSION;
   init_params.vm_isolate_snapshot = vm_isolate_snapshot_buffer;
   init_params.instructions_snapshot = instructions_snapshot;
   init_params.data_snapshot = data_snapshot;
   init_params.create = CreateIsolateAndSetup;
   init_params.shutdown = ShutdownIsolate;
   init_params.file_open = DartUtils::OpenFile;
@@ skipping 54 matching lines @@
   Platform::Exit(Process::GlobalExitCode());
 }
 
 }  // namespace bin
 }  // namespace dart
 
 int main(int argc, char** argv) {
   dart::bin::main(argc, argv);
   UNREACHABLE();
 }