| OLD | NEW |
| 1 #!/usr/bin/env dart | 1 #!/usr/bin/env dart |
| 2 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file | 2 // Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file |
| 3 // for details. All rights reserved. Use of this source code is governed by a | 3 // for details. All rights reserved. Use of this source code is governed by a |
| 4 // BSD-style license that can be found in the LICENSE file. | 4 // BSD-style license that can be found in the LICENSE file. |
| 5 | 5 |
| 6 /// Command line tool to merge the SDK libraries and our patch files. | 6 /// Command line tool to merge the SDK libraries and our patch files. |
| 7 /// This is currently designed as an offline tool, but we could automate it. | 7 /// This is currently designed as an offline tool, but we could automate it. |
| 8 | 8 |
| 9 import 'dart:io'; | 9 import 'dart:io'; |
| 10 import 'dart:isolate' show RawReceivePort; | 10 import 'dart:isolate' show RawReceivePort; |
| 11 import 'dart:async'; | 11 import 'dart:async'; |
| 12 import 'dart:math' as math; | 12 import 'dart:math' as math; |
| 13 import 'dart:convert' show JSON; | 13 import 'dart:convert' show JSON; |
| 14 | 14 |
| 15 import 'package:analyzer/analyzer.dart'; | 15 import 'package:analyzer/analyzer.dart'; |
| 16 import 'package:analyzer/src/generated/sdk.dart'; | 16 import 'package:analyzer/src/generated/sdk.dart'; |
| 17 import 'package:path/path.dart' as path; | 17 import 'package:path/path.dart' as path; |
| 18 | 18 |
| 19 import 'package:front_end/front_end.dart'; | 19 import 'package:front_end/src/fasta/fasta.dart' as fasta |
| 20 import 'package:front_end/src/base/processed_options.dart'; | 20 show compile, compilePlatform, writeDepsFile; |
| 21 import 'package:front_end/src/kernel_generator_impl.dart'; | |
| 22 import 'package:front_end/src/fasta/util/relativize.dart' show relativizeUri; | |
| 23 | 21 |
| 24 import 'package:front_end/src/fasta/fasta.dart' as fasta show getDependencies; | 22 import 'package:compiler/src/kernel/fasta_support.dart' as dart2js |
| 25 import 'package:front_end/src/fasta/kernel/utils.dart' show writeProgramToFile; | 23 show compilePlatform; |
| 26 | |
| 27 import 'package:kernel/target/targets.dart'; | |
| 28 import 'package:kernel/target/vm_fasta.dart'; | |
| 29 import 'package:kernel/target/flutter_fasta.dart'; | |
| 30 import 'package:compiler/src/kernel/dart2js_target.dart' show Dart2jsTarget; | |
| 31 | 24 |
| 32 /// Set of input files that were read by this script to generate patched SDK. | 25 /// Set of input files that were read by this script to generate patched SDK. |
| 33 /// We will dump it out into the depfile for ninja to use. | 26 /// We will dump it out into the depfile for ninja to use. |
| 34 /// | 27 /// |
| 35 /// For more information see GN and Ninja references: | 28 /// For more information see GN and Ninja references: |
| 36 /// https://chromium.googlesource.com/chromium/src/+/56807c6cb383140af0c03da8
f6731d77785d7160/tools/gn/docs/reference.md#depfile_string_File-name-for-input-d
ependencies-for-actions | 29 /// https://chromium.googlesource.com/chromium/src/+/56807c6cb383140af0c03da8
f6731d77785d7160/tools/gn/docs/reference.md#depfile_string_File-name-for-input-d
ependencies-for-actions |
| 37 /// https://ninja-build.org/manual.html#_depfile | 30 /// https://ninja-build.org/manual.html#_depfile |
| 38 /// | 31 /// |
| 39 final deps = new Set<Uri>(); | 32 final deps = new Set<Uri>(); |
| 40 | 33 |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 119 _copyExtraLibraries(sdkOut, locations); | 112 _copyExtraLibraries(sdkOut, locations); |
| 120 | 113 |
| 121 Uri platform = outDirUri.resolve('platform.dill.tmp'); | 114 Uri platform = outDirUri.resolve('platform.dill.tmp'); |
| 122 Uri outline = outDirUri.resolve('outline.dill'); | 115 Uri outline = outDirUri.resolve('outline.dill'); |
| 123 Uri librariesJson = outDirUri.resolve("lib/libraries.json"); | 116 Uri librariesJson = outDirUri.resolve("lib/libraries.json"); |
| 124 Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile)); | 117 Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile)); |
| 125 | 118 |
| 126 await _writeSync( | 119 await _writeSync( |
| 127 librariesJson.toFilePath(), JSON.encode({"libraries": locations})); | 120 librariesJson.toFilePath(), JSON.encode({"libraries": locations})); |
| 128 | 121 |
| 129 var flags = new TargetFlags(); | 122 if (forVm || forFlutter) { |
| 130 var target = forVm | 123 await fasta.compilePlatform(outDirUri, platform, |
| 131 ? new VmFastaTarget(flags) | 124 packages: packages, |
| 132 : (forFlutter ? new FlutterFastaTarget(flags) : new Dart2jsTarget(flags)); | 125 outlineOutput: outline, |
| 133 var platformDeps = | 126 backendTarget: forVm ? 'vm_fasta' : 'flutter_fasta'); |
| 134 await compilePlatform(outDirUri, target, packages, platform, outline); | 127 } else { |
| 135 deps.addAll(platformDeps); | 128 await dart2js.compilePlatform(outDirUri, platform, |
| 129 packages: packages, outlineOutput: outline); |
| 130 } |
| 136 | 131 |
| 137 if (forVm) { | 132 if (forVm) { |
| 138 // TODO(sigmund): add support for the flutter vmservice_sky as well. | 133 var base = path.fromUri(Platform.script); |
| 134 Uri repositoryDir = |
| 135 new Uri.directory(path.dirname(path.dirname(path.absolute(base)))); |
| 139 var vmserviceName = 'vmservice_io'; | 136 var vmserviceName = 'vmservice_io'; |
| 140 var base = path.fromUri(Platform.script); | 137 Uri vmserviceSdk = repositoryDir.resolve('runtime/bin/vmservice_sdk/'); |
| 141 Uri dartDir = | |
| 142 new Uri.directory(path.dirname(path.dirname(path.absolute(base)))); | |
| 143 var program = await kernelForProgram( | |
| 144 Uri.parse('dart:$vmserviceName'), | |
| 145 new CompilerOptions() | |
| 146 ..sdkSummary = outline | |
| 147 ..dartLibraries = <String, Uri>{ | |
| 148 '_vmservice': outDirUri.resolve('lib/vmservice/vmservice.dart'), | |
| 149 'vmservice_io': | |
| 150 dartDir.resolve('runtime/bin/vmservice/vmservice_io.dart'), | |
| 151 } | |
| 152 ..packagesFileUri = packages); | |
| 153 Uri vmserviceUri = outDirUri.resolve('$vmserviceName.dill'); | 138 Uri vmserviceUri = outDirUri.resolve('$vmserviceName.dill'); |
| 154 await writeProgramToFile(program, vmserviceUri); | 139 // TODO(sigmundch): Specify libraries.json directly instead of "--sdk" |
| 140 // after #29882 is fixed. |
| 141 await fasta.compile([ |
| 142 "--sdk=$vmserviceSdk", |
| 143 "--platform=$outline", |
| 144 "--target=vm_fasta", |
| 145 "--packages=$packages", |
| 146 "dart:$vmserviceName", |
| 147 "-o", |
| 148 "$vmserviceUri", |
| 149 ]); |
| 155 } | 150 } |
| 156 | 151 |
| 157 Uri platformFinalLocation = outDirUri.resolve('platform.dill'); | 152 Uri platformFinalLocation = outDirUri.resolve('platform.dill'); |
| 158 | 153 |
| 159 // We generate a dependency file for GN to properly regenerate the patched sdk | 154 // We generate a dependency file for GN to properly regenerate the patched sdk |
| 160 // folder, outline.dill and platform.dill files when necessary: either when | 155 // folder, outline.dill and platform.dill files when necessary: either when |
| 161 // the sdk sources change or when this script is updated. In particular: | 156 // the sdk sources change or when this script is updated. In particular: |
| 162 // | 157 // |
| 163 // - sdk changes: we track the actual sources we are compiling. If we are | 158 // - sdk changes: we track the actual sources we are compiling. If we are |
| 164 // building the dart2js sdk, this includes the dart2js-specific patch | 159 // building the dart2js sdk, this includes the dart2js-specific patch |
| 165 // files. | 160 // files. |
| 166 // | 161 // |
| 167 // These files are tracked by [deps] and passed below to [writeDepsFile] in | 162 // These files are tracked by [deps] and passed below to [writeDepsFile] in |
| 168 // the extraDependencies argument. | 163 // the extraDependencies argument. |
| 169 // | 164 // |
| 170 // - script updates: we track this script file and any code it imports (even | 165 // - script updates: we track this script file and any code it imports (even |
| 171 // sdk libraries). Note that this script runs on the standalone VM, so any | 166 // sdk libraries). Note that this script runs on the standalone VM, so any |
| 172 // sdk library used by this script indirectly depends on a VM-specific | 167 // sdk library used by this script indirectly depends on a VM-specific |
| 173 // patch file. | 168 // patch file. |
| 174 // | 169 // |
| 175 // These set of files is discovered by `getDependencies` below, and the | 170 // These set of files is discovered by `writeDepsFile` below, and the |
| 176 // [platformForDeps] is always the VM-specific `platform.dill` file. | 171 // [platformForDeps] is always the VM-specific `platform.dill` file. |
| 172 // |
| 173 // TODO(sigmund): we should change this: |
| 174 // - we should rewrite writeDepsFile: fasta could provide an API to crawl |
| 175 // the dependencies, but anything that is GN specific, should be on |
| 176 // this file instead. |
| 177 // |
| 178 // - We don't need to include sdk dependencies of the script because |
| 179 // those are already included indirectly (either in [deps] when |
| 180 // building the sdk for the VM, or via the .GN dependencies in the |
| 181 // build files for dart2js and flutter). |
| 177 var platformForDeps = platform; | 182 var platformForDeps = platform; |
| 178 var sdkDir = outDirUri; | 183 var sdkDir = outDirUri; |
| 179 if (forDart2js || forFlutter) { | 184 if (forDart2js || forFlutter) { |
| 180 // Note: this would fail if `../patched_sdk/platform.dill` doesn't exist. We | 185 // Note: this would fail if `../patched_sdk/platform.dill` doesn't exist. We |
| 181 // added an explicit dependency in the .GN rules so patched_dart2js_sdk (and | 186 // added an explicit dependency in the .GN rules so patched_dart2js_sdk (and |
| 182 // patched_flutter_sdk) depend on patched_sdk to ensure that it exists. | 187 // patched_flutter_sdk) depend on patched_sdk to ensure that it exists. |
| 183 platformForDeps = outDirUri.resolve('../patched_sdk/platform.dill'); | 188 platformForDeps = outDirUri.resolve('../patched_sdk/platform.dill'); |
| 184 sdkDir = outDirUri.resolve('../patched_sdk/'); | 189 sdkDir = outDirUri.resolve('../patched_sdk/'); |
| 185 } | 190 } |
| 186 deps.addAll(await fasta.getDependencies(Platform.script, | 191 await fasta.writeDepsFile(Platform.script, |
| 187 sdk: sdkDir, packages: packages, platform: platformForDeps)); | 192 Uri.base.resolveUri(new Uri.file("$outDir.d")), platformFinalLocation, |
| 188 await writeDepsFile(Uri.base.resolveUri(new Uri.file("$outDir.d")), | 193 sdk: sdkDir, |
| 189 platformFinalLocation, deps); | 194 packages: packages, |
| 195 platform: platformForDeps, |
| 196 extraDependencies: deps); |
| 197 |
| 190 await new File.fromUri(platform).rename(platformFinalLocation.toFilePath()); | 198 await new File.fromUri(platform).rename(platformFinalLocation.toFilePath()); |
| 191 } | 199 } |
| 192 | 200 |
| 193 /// Generates an outline.dill and platform.dill file containing the result of | |
| 194 /// compiling a platform's SDK. | |
| 195 /// | |
| 196 /// Returns a list of dependencies read by the compiler. This list can be used | |
| 197 /// to create GN dependency files. | |
| 198 Future<List<Uri>> compilePlatform(Uri patchedSdk, Target target, Uri packages, | |
| 199 Uri fullOutput, Uri outlineOutput) async { | |
| 200 var options = new CompilerOptions() | |
| 201 ..strongMode = false | |
| 202 ..compileSdk = true | |
| 203 ..sdkRoot = patchedSdk | |
| 204 ..packagesFileUri = packages | |
| 205 ..chaseDependencies = true | |
| 206 ..target = target; | |
| 207 | |
| 208 var result = await generateKernel( | |
| 209 new ProcessedOptions( | |
| 210 options, | |
| 211 // TODO(sigmund): pass all sdk libraries needed here, and make this | |
| 212 // hermetic. | |
| 213 false, | |
| 214 [Uri.parse('dart:core')]), | |
| 215 buildSummary: true, | |
| 216 buildProgram: true); | |
| 217 new File.fromUri(outlineOutput).writeAsBytesSync(result.summary); | |
| 218 await writeProgramToFile(result.program, fullOutput); | |
| 219 return result.deps; | |
| 220 } | |
| 221 | |
| 222 Future writeDepsFile( | |
| 223 Uri output, Uri depsFile, Iterable<Uri> allDependencies) async { | |
| 224 if (allDependencies.isEmpty) return; | |
| 225 String toRelativeFilePath(Uri uri) { | |
| 226 // Ninja expects to find file names relative to the current working | |
| 227 // directory. We've tried making them relative to the deps file, but that | |
| 228 // doesn't work for downstream projects. Making them absolute also | |
| 229 // doesn't work. | |
| 230 // | |
| 231 // We can test if it works by running ninja twice, for example: | |
| 232 // | |
| 233 // ninja -C xcodebuild/ReleaseX64 runtime_kernel -d explain | |
| 234 // ninja -C xcodebuild/ReleaseX64 runtime_kernel -d explain | |
| 235 // | |
| 236 // The second time, ninja should say: | |
| 237 // | |
| 238 // ninja: Entering directory `xcodebuild/ReleaseX64' | |
| 239 // ninja: no work to do. | |
| 240 // | |
| 241 // It's broken if it says something like this: | |
| 242 // | |
| 243 // ninja explain: expected depfile 'patched_sdk.d' to mention | |
| 244 // 'patched_sdk/platform.dill', got | |
| 245 // '/.../xcodebuild/ReleaseX64/patched_sdk/platform.dill' | |
| 246 return Uri.parse(relativizeUri(uri, base: Uri.base)).toFilePath(); | |
| 247 } | |
| 248 | |
| 249 StringBuffer sb = new StringBuffer(); | |
| 250 sb.write(toRelativeFilePath(output)); | |
| 251 sb.write(":"); | |
| 252 for (Uri uri in allDependencies) { | |
| 253 sb.write(" "); | |
| 254 sb.write(toRelativeFilePath(uri)); | |
| 255 } | |
| 256 sb.writeln(); | |
| 257 await new File.fromUri(depsFile).writeAsString("$sb"); | |
| 258 } | |
| 259 | |
| 260 /// Updates the contents of | 201 /// Updates the contents of |
| 261 /// sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart to include | 202 /// sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart to include |
| 262 /// declarations for vm internal libraries. | 203 /// declarations for vm internal libraries. |
| 263 String _updateLibraryMetadata(String sdkOut, String libContents) { | 204 String _updateLibraryMetadata(String sdkOut, String libContents) { |
| 264 if (!forVm && !forFlutter) return libContents; | 205 if (!forVm && !forFlutter) return libContents; |
| 265 var extraLibraries = new StringBuffer(); | 206 var extraLibraries = new StringBuffer(); |
| 266 extraLibraries.write(''' | 207 extraLibraries.write(''' |
| 267 "_builtin": const LibraryInfo( | 208 "_builtin": const LibraryInfo( |
| 268 "_builtin/_builtin.dart", | 209 "_builtin/_builtin.dart", |
| 269 categories: "Client,Server", | 210 categories: "Client,Server", |
| (...skipping 463 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 733 if (diff != 0) return diff; | 674 if (diff != 0) return diff; |
| 734 return end - other.end; | 675 return end - other.end; |
| 735 } | 676 } |
| 736 } | 677 } |
| 737 | 678 |
| 738 List<SdkLibrary> _getSdkLibraries(String contents) { | 679 List<SdkLibrary> _getSdkLibraries(String contents) { |
| 739 var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(forDart2js); | 680 var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(forDart2js); |
| 740 parseCompilationUnit(contents).accept(libraryBuilder); | 681 parseCompilationUnit(contents).accept(libraryBuilder); |
| 741 return libraryBuilder.librariesMap.sdkLibraries; | 682 return libraryBuilder.librariesMap.sdkLibraries; |
| 742 } | 683 } |
| OLD | NEW |