dart2js: Use the correct hashes for deferred code in the startup-emitter.

pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart2js.js_emitter.startup_emitter.model_emitter;
 
+import 'dart:convert' show JsonEncoder;
+
+import '../../common.dart';
+
 import '../../constants/values.dart' show ConstantValue, FunctionConstantValue;
 import '../../dart2jslib.dart' show Compiler;
 import '../../elements/elements.dart' show ClassElement, FunctionElement;
+import '../../hash/sha1.dart' show Hasher;
+
+import '../../io/code_output.dart';
+
+import '../../io/line_column_provider.dart' show
+    LineColumnCollector,
+    LineColumnProvider;
+
+import '../../io/source_map_builder.dart' show
+    SourceMapBuilder;
+
 import '../../js/js.dart' as js;
 import '../../js_backend/js_backend.dart' show
     JavaScriptBackend,
     Namer,
     ConstantEmitter;
 
+import '../../util/util.dart' show
+    NO_LOCATION_SPANNABLE;
+
+import '../../util/uri_extras.dart' show
+    relativize;
+
 import '../js_emitter.dart' show AstContainer, NativeEmitter;
 
 import 'package:js_runtime/shared/embedded_names.dart' show
     CREATE_NEW_ISOLATE,
     DEFERRED_INITIALIZED,
     DEFERRED_LIBRARY_URIS,
     DEFERRED_LIBRARY_HASHES,
     GET_TYPE_FROM_NAME,
     INITIALIZE_LOADED_HUNK,
     INTERCEPTORS_BY_TAG,
     IS_HUNK_INITIALIZED,
     IS_HUNK_LOADED,
     LEAF_TAGS,
     MANGLED_GLOBAL_NAMES,
     METADATA,
     NATIVE_SUPERCLASS_TAG_NAME,
     TYPE_TO_INTERCEPTOR_MAP,
     TYPES;
 
 import '../js_emitter.dart' show NativeGenerator, buildTearOffCode;
 import '../model.dart';
 
+part 'deferred_fragment_hash.dart';
 part 'fragment_emitter.dart';
 
 class ModelEmitter {
   final Compiler compiler;
   final Namer namer;
   ConstantEmitter constantEmitter;
   final NativeEmitter nativeEmitter;
+  final bool shouldGenerateSourceMap;
+
+  // The full code that is written to each hunk part-file.
+  final Map<Fragment, CodeOutput> outputBuffers = <Fragment, CodeOutput>{};
+
 
   JavaScriptBackend get backend => compiler.backend;
 
   /// For deferred loading we communicate the initializers via this global var.
   static const String deferredInitializersGlobal =
       r"$__dart_deferred_initializers__";
 
   static const String deferredExtension = "part.js";
 
   static const String typeNameProperty = r"builtin$cls";
 
-  ModelEmitter(Compiler compiler, Namer namer, this.nativeEmitter)
+  ModelEmitter(Compiler compiler, Namer namer, this.nativeEmitter,
+               this.shouldGenerateSourceMap)
       : this.compiler = compiler,
         this.namer = namer {
     this.constantEmitter = new ConstantEmitter(
         compiler, namer, this.generateConstantReference,
         constantListGenerator);
   }
 
   js.Expression constantListGenerator(js.Expression array) {
     // TODO(floitsch): remove hard-coded name.
     return js.js('makeConstList(#)', [array]);
@@ skipping 52 matching lines @@
     // We are only interested in the "isInlined" part, but it does not hurt to
     // test for the other predicates.
     if (isConstantInlinedOrAlreadyEmitted(value)) {
       return constantEmitter.generate(value);
     }
     return js.js('#.#', [namer.globalObjectForConstant(value),
     namer.constantName(value)]);
   }
 
   int emitProgram(Program program) {
-    List<Fragment> fragments = program.fragments;
-    MainFragment mainFragment = fragments.first;
-
-    int totalSize = 0;
+    MainFragment mainFragment = program.fragments.first;
+    List<DeferredFragment> deferredFragments =
+        new List<DeferredFragment>.from(program.fragments.skip(1));
 
     FragmentEmitter fragmentEmitter =
         new FragmentEmitter(compiler, namer, backend, constantEmitter, this);
 
-    // We have to emit the deferred fragments first, since we need their
-    // deferred hash (which depends on the output) when emitting the main
-    // fragment.
-    List<js.Expression> fragmentsCode = fragments.skip(1).map(
-            (DeferredFragment deferredFragment) {
-          js.Expression types =
-          program.metadataTypesForOutputUnit(deferredFragment.outputUnit);
-          return fragmentEmitter.emitDeferredFragment(
-              deferredFragment, types, program.holders);
-        }).toList();
-
-    js.Statement mainAst = fragmentEmitter.emitMainFragment(program);
+    Map<DeferredFragment, _DeferredFragmentHash> deferredHashTokens =
+      new Map<DeferredFragment, _DeferredFragmentHash>();
+    for (DeferredFragment fragment in deferredFragments) {
+      deferredHashTokens[fragment] = new _DeferredFragmentHash(fragment);
+    }
+
+    js.Statement mainCode =
+        fragmentEmitter.emitMainFragment(program, deferredHashTokens);
+
+    Map<DeferredFragment, js.Expression> deferredFragmentsCode =
+        <DeferredFragment, js.Expression>{};
+
+    for (DeferredFragment fragment in deferredFragments) {
+      js.Expression types =
+          program.metadataTypesForOutputUnit(fragment.outputUnit);
+      deferredFragmentsCode[fragment] = fragmentEmitter.emitDeferredFragment(
+                fragment, types, program.holders);
+    }
 
     js.TokenCounter counter = new js.TokenCounter();
-    fragmentsCode.forEach(counter.countTokens);
-    counter.countTokens(mainAst);
+    deferredFragmentsCode.values.forEach(counter.countTokens);
+    counter.countTokens(mainCode);
 
     program.finalizers.forEach((js.TokenFinalizer f) => f.finalizeTokens());
 
-    for (int i = 0; i < fragmentsCode.length; ++i) {
-      String code = js.prettyPrint(fragmentsCode[i], compiler).getText();
-      totalSize += code.length;
-      compiler.outputProvider(fragments[i+1].outputFileName, deferredExtension)
-        ..add(code)
-        ..close();
-    }
-
-    String mainCode = js.prettyPrint(mainAst, compiler).getText();
-    compiler.outputProvider(mainFragment.outputFileName, 'js')
-      ..add(buildGeneratedBy(compiler))
-      ..add(mainCode)
-      ..close();
-    totalSize += mainCode.length;
-
-    return totalSize;
-  }
-
+    Map<DeferredFragment, String> hunkHashes =
+        writeDeferredFragments(deferredFragmentsCode);

[Siggi Cherem (dart-lang), 2015/07/23 00:01:10]

will writeDefferredFragments be used anywhere else?

If not, maybe just pass down the map of Fragment->FragmentHash and set the hash
directly there.

[floitsch, 2015/07/29 18:00:28]

Whenever possible I try to return the result of a function as return-value. I
really don't like when functions have side-effects that aren't obvious (from
their name).
I could rename the function to `writeDeferredFragmentsAndFillHunkMap` but in
this case we can return the map, so there isn't any need for it.

+
+    // Now that we have written the deferred hunks, we can update the hash
+    // tokens in the main-fragment.
+    deferredHashTokens.forEach((DeferredFragment key,
+                                _DeferredFragmentHash token) {
+      token.setHash(hunkHashes[key]);
+    });
+
+    writeMainFragment(mainFragment, mainCode);
+
+    if (backend.requiresPreamble &&
+        !backend.htmlLibraryIsLoaded) {
+      compiler.reportHint(NO_LOCATION_SPANNABLE, MessageKind.PREAMBLE);
+    }
+
+    // Return the total program size.
+    return outputBuffers.values.fold(0, (a, b) => a + b.length);
+  }
+
+  /// Generates a simple header that provides the compiler's build id.
   String buildGeneratedBy(compiler) {
     var suffix = '';
     if (compiler.hasBuildId) suffix = ' version: ${compiler.buildId}';
     return '// Generated by dart2js (fast startup), '
         'the Dart to JavaScript compiler$suffix.\n';
   }
+
+  /// Writes all deferred fragment's code into files.
+  ///
+  /// Returns a map from fragment to its hashcode (as used for the deferred
+  /// library code).
+  ///
+  /// Updates the shared [outputBuffers] field with the output.
+  Map<DeferredFragment, String> writeDeferredFragments(
+      Map<DeferredFragment, js.Expression> fragmentsCode) {
+    Map<DeferredFragment, String> hunkHashes = <DeferredFragment, String>{};
+
+    fragmentsCode.forEach((DeferredFragment fragment, js.Expression code) {
+      hunkHashes[fragment] = writeDeferredFragment(fragment, code);
+    });
+
+    return hunkHashes;
+  }
+
+  // Writes the given [fragment]'s [code] into a file.
+  //
+  // Updates the shared [outputBuffers] field with the output.
+  void writeMainFragment(MainFragment fragment, js.Statement code) {
+    LineColumnCollector lineColumnCollector;
+    List<CodeOutputListener> codeOutputListeners;
+    if (shouldGenerateSourceMap) {
+      lineColumnCollector = new LineColumnCollector();
+      codeOutputListeners = <CodeOutputListener>[lineColumnCollector];
+    }
+
+    CodeOutput mainOutput = new StreamCodeOutput(
+            compiler.outputProvider('', 'js'),
+            codeOutputListeners);
+    outputBuffers[fragment] = mainOutput;
+
+    mainOutput.addBuffer(js.prettyPrint(code, compiler,
+        monitor: compiler.dumpInfoTask));
+
+    if (compiler.deferredMapUri != null) {

[Siggi Cherem (dart-lang), 2015/07/23 00:01:10]

any reason we need to emit the map here in the js_emitter? This feels similar to
dump-info, in that it could either be emitted directly by the deferredLoadTask
or the compiler itself.

[floitsch, 2015/07/29 18:00:28]

No reason. A lot of this code has been copied/inspired by the full-emitter.
I guess it makes sense to keep it in this file, since we emit almost everything
here. (Note that the startup-emitter doesn't support dump-info yet).

For now I moved it to `emitProgram`. The `writeMainFragment` was not the right
place.

+      writeDeferredMap();
+    }
+
+    if (shouldGenerateSourceMap) {
+      mainOutput.add(
+          generateSourceMapTag(compiler.sourceMapUri, compiler.outputUri));
+    }
+
+    mainOutput.close();
+
+    if (shouldGenerateSourceMap) {
+      outputSourceMap(mainOutput, lineColumnCollector, '',
+      compiler.sourceMapUri, compiler.outputUri);
+    }
+  }
+
+  // Writes the given [fragment]'s [code] into a file.
+  //
+  // Returns the deferred fragment's hash.
+  //
+  // Updates the shared [outputBuffers] field with the output.
+  String writeDeferredFragment(DeferredFragment fragment, js.Expression code) {
+    List<CodeOutputListener> outputListeners = <CodeOutputListener>[];
+    Hasher hasher = new Hasher();
+    outputListeners.add(hasher);
+
+    LineColumnCollector lineColumnCollector;
+    if (shouldGenerateSourceMap) {
+      lineColumnCollector = new LineColumnCollector();
+      outputListeners.add(lineColumnCollector);
+    }
+
+    String hunkPrefix = fragment.outputFileName;
+
+    CodeOutput output = new StreamCodeOutput(
+        compiler.outputProvider(hunkPrefix, deferredExtension),
+        outputListeners);
+
+    outputBuffers[fragment] = output;
+
+    // The [code] contains the function that must be invoked when the deferred
+    // hunk is loaded.
+    // That function must be in a map from its hashcode to the function. Since
+    // we don't know the hash before we actually emit the code we store the
+    // function in a temporary field first:
+    //
+    //   deferredInitializer.current = <pretty-printed code>;
+    //   deferredInitializer[<hash>] = deferredInitializer.current;
+
+    output.add('\n${deferredInitializersGlobal}.current = ');
+
+    output.addBuffer(js.prettyPrint(code, compiler,
+        monitor: compiler.dumpInfoTask));
+
+    // Make a unique hash of the code (before the sourcemaps are added)
+    // This will be used to retrieve the initializing function from the global
+    // variable.
+    String hash = hasher.getHash();
+
+    // Now we copy the deferredInitializer.current into its correct hash.
+    output.add('\n${deferredInitializersGlobal}["$hash"] = '
+        '${deferredInitializersGlobal}.current');
+
+    if (shouldGenerateSourceMap) {
+      Uri mapUri, partUri;
+      Uri sourceMapUri = compiler.sourceMapUri;
+      Uri outputUri = compiler.outputUri;
+
+      if (sourceMapUri != null) {
+        String mapFileName =
+            hunkPrefix + deferredExtension + ".map";
+        List<String> mapSegments = sourceMapUri.pathSegments.toList();
+        mapSegments[mapSegments.length - 1] = mapFileName;
+        mapUri = compiler.sourceMapUri.replace(pathSegments: mapSegments);
+      }
+
+      if (outputUri != null) {
+        String hunkFileName = hunkPrefix + deferredExtension;
+        List<String> partSegments = outputUri.pathSegments.toList();
+        partSegments[partSegments.length - 1] = hunkFileName;
+        partUri = compiler.outputUri.replace(pathSegments: partSegments);
+      }
+
+      output.add(generateSourceMapTag(mapUri, partUri));
+      output.close();
+      outputSourceMap(output, lineColumnCollector, hunkPrefix, mapUri, partUri);
+    } else {
+      output.close();
+    }
+
+    return hash;
+  }
+
+  String generateSourceMapTag(Uri sourceMapUri, Uri fileUri) {
+    if (sourceMapUri != null && fileUri != null) {
+      String sourceMapFileName = relativize(fileUri, sourceMapUri, false);
+      return '''
+
+//# sourceMappingURL=$sourceMapFileName
+''';
+    }
+    return '';
+  }
+
+
+  void outputSourceMap(CodeOutput output,
+                       LineColumnProvider lineColumnProvider,
+                       String name,
+                       [Uri sourceMapUri,
+                       Uri fileUri]) {
+    if (!shouldGenerateSourceMap) return;
+    // Create a source file for the compilation output. This allows using
+    // [:getLine:] to transform offsets to line numbers in [SourceMapBuilder].
+    SourceMapBuilder sourceMapBuilder =
+    new SourceMapBuilder(sourceMapUri, fileUri, lineColumnProvider);
+    output.forEachSourceLocation(sourceMapBuilder.addMapping);
+    String sourceMap = sourceMapBuilder.build();
+    compiler.outputProvider(name, 'js.map')
+      ..add(sourceMap)
+      ..close();
+  }
+
+  /// Writes a mapping from library-name to hunk files.
+  ///
+  /// The output is written into a separate file that can be used by outside
+  /// tools.
+  void writeDeferredMap() {
+    Map<String, dynamic> mapping = new Map<String, dynamic>();
+    // Json does not support comments, so we embed the explanation in the
+    // data.
+    mapping["_comment"] = "This mapping shows which compiled `.js` files are "
+        "needed for a given deferred library import.";
+    mapping.addAll(compiler.deferredLoadTask.computeDeferredMap());
+    compiler.outputProvider(compiler.deferredMapUri.path, 'deferred_map')
+      ..add(const JsonEncoder.withIndent("  ").convert(mapping))
+      ..close();
+  }
 }