Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(43)

Side by Side Diff: tools/gn/toolchain_manager.cc

Issue 56433003: GN threading refactor (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/gn/toolchain_manager.h ('k') | tools/gn/trace.h » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "tools/gn/toolchain_manager.h"
6
7 #include <set>
8
9 #include "base/bind.h"
10 #include "build/build_config.h"
11 #include "tools/gn/err.h"
12 #include "tools/gn/item.h"
13 #include "tools/gn/item_node.h"
14 #include "tools/gn/item_tree.h"
15 #include "tools/gn/parse_tree.h"
16 #include "tools/gn/scheduler.h"
17 #include "tools/gn/scope.h"
18 #include "tools/gn/scope_per_file_provider.h"
19 #include "tools/gn/trace.h"
20
21 // How toolchain loading works
22 // ---------------------------
23 // When we start loading a build, we'll load the build config file and that
24 // will call set_default_toolchain. We'll schedule a load of the file
25 // containing the default toolchain definition, and can do this in parallel
26 // with all other build files. Targets will have an implicit dependency on the
27 // toolchain so we won't write out any files until we've loaded the toolchain
28 // definition.
29 //
30 // When we see a reference to a target using a different toolchain, it gets
31 // more complicated. In this case, the toolchain definition contains arguments
32 // to pass into the build config file when it is invoked in the context of that
33 // toolchain. As a result, we have to actually see the definition of the
34 // toolchain before doing anything else.
35 //
36 // So when we see a reference to a non-default toolchain we do the following:
37 //
38 // 1. Schedule a load of the file containing the toolchain definition, if it
39 // isn't loaded already.
40 // 2. When the toolchain definition is loaded, we schedule a load of the
41 // build config file in the context of that toolchain. We'll use the
42 // arguments from the toolchain definition to execute it.
43 // 3. When the build config is set up, then we can load all of the individual
44 // buildfiles in the context of that config that we require.
45
46 namespace {
47
48 enum ToolchainState {
49 // Toolchain settings have not requested to be loaded. This means we
50 // haven't seen any targets that require this toolchain yet. This means that
51 // we have seen a toolchain definition, but no targets that use it. Not
52 // loading the settings automatically allows you to define a bunch of
53 // toolchains and potentially not use them without much overhead.
54 TOOLCHAIN_NOT_LOADED,
55
56 // The toolchain definition for non-default toolchains has been scheduled
57 // to be loaded but has not completed. When this is done, we can load the
58 // settings file. This is needed to get the arguments out of the toolchain
59 // definition. This is skipped for the default toolchain which has no
60 // arguments (see summary above).
61 TOOLCHAIN_DEFINITION_LOADING,
62
63 // The settings have been scheduled to be loaded but have not completed.
64 TOOLCHAIN_SETTINGS_LOADING,
65
66 // The settings are done being loaded.
67 TOOLCHAIN_SETTINGS_LOADED
68 };
69
70 SourceFile DirToBuildFile(const SourceDir& dir) {
71 return SourceFile(dir.value() + "BUILD.gn");
72 }
73
74 } // namespace
75
76 struct ToolchainManager::Info {
77 Info(const BuildSettings* build_settings,
78 const Label& toolchain_name,
79 const std::string& output_subdir_name)
80 : state(TOOLCHAIN_NOT_LOADED),
81 settings(build_settings, output_subdir_name),
82 toolchain(new Toolchain(&settings, toolchain_name)),
83 toolchain_set(false),
84 toolchain_file_loaded(false),
85 item_node(NULL) {
86 settings.set_toolchain_label(toolchain_name);
87 }
88
89 ~Info() {
90 if (!item_node) // See toolchain definition for more.
91 delete toolchain;
92 }
93
94 // Makes sure that an ItemNode is created for the toolchain, which lets
95 // targets depend on the (potentially future) loading of the toolchain.
96 //
97 // We can't always do this at the beginning since when doing the default
98 // build config, we don't know the toolchain name yet. We also need to go
99 // through some effort to avoid doing this inside the toolchain manager's
100 // lock (to avoid holding two locks at once).
101 void EnsureItemNode() {
102 if (!item_node) {
103 ItemTree& tree = settings.build_settings()->item_tree();
104 item_node = new ItemNode(toolchain);
105 tree.AddNodeLocked(item_node);
106 }
107 }
108
109 ToolchainState state;
110
111 // The first place in the build that we saw a reference for this toolchain.
112 // This allows us to report errors if it can't be loaded and blame some
113 // reasonable place of the code. This will be empty for the default toolchain.
114 LocationRange specified_from;
115
116 // When the state is TOOLCHAIN_SETTINGS_LOADED, the settings should be
117 // considered read-only and can be read without locking. Otherwise, they
118 // should not be accessed at all except to load them (which can therefore
119 // also be done outside of the lock). This works as long as the state flag
120 // is only ever read or written inside the lock.
121 Settings settings;
122
123 // When we create an item node, this pointer will be owned by that node
124 // so it's lifetime is managed by the dependency graph. Before we've created
125 // the ItemNode, this class has to takre responsibility for this pointer.
126 Toolchain* toolchain;
127 bool toolchain_set;
128 LocationRange toolchain_definition_location;
129
130 // Set when the file corresponding to the toolchain definition is loaded.
131 // This will normally be set right after "toolchain_set". However, if the
132 // toolchain definition is missing, the file might be marked loaded but the
133 // toolchain definition could still be unset.
134 bool toolchain_file_loaded;
135
136 // While state == TOOLCHAIN_SETTINGS_LOADING || TOOLCHAIN_DEFINITION_LOADING,
137 // this will collect all scheduled invocations using this toolchain. They'll
138 // be issued once the settings file has been interpreted.
139 //
140 // The map maps the source file to "some" location it was invoked from (so
141 // we can give good error messages). It does NOT map to the root of the
142 // file to be invoked (the file still needs loading). This will be NULL
143 // for internally invoked files.
144 typedef std::map<SourceFile, LocationRange> ScheduledInvocationMap;
145 ScheduledInvocationMap scheduled_invocations;
146
147 // Tracks all scheduled and executed invocations for this toolchain. This
148 // is used to avoid invoking a file more than once for a toolchain.
149 std::set<SourceFile> all_invocations;
150
151 // Filled in by EnsureItemNode, see that for more.
152 ItemNode* item_node;
153 };
154
155 ToolchainManager::ToolchainManager(const BuildSettings* build_settings)
156 : build_settings_(build_settings) {
157 }
158
159 ToolchainManager::~ToolchainManager() {
160 for (ToolchainMap::iterator i = toolchains_.begin();
161 i != toolchains_.end(); ++i)
162 delete i->second;
163 toolchains_.clear();
164 }
165
166 void ToolchainManager::StartLoadingUnlocked(const SourceFile& build_file_name) {
167 // How the default build config works: Initially we don't have a toolchain
168 // name to call the settings for the default build config. So we create one
169 // with an empty toolchain name and execute the default build config file.
170 // When that's done, we'll go and fix up the name to the default build config
171 // that the script set.
172 base::AutoLock lock(GetLock());
173 Err err;
174 Info* info = LoadNewToolchainLocked(LocationRange(), Label(), &err);
175 if (err.has_error())
176 g_scheduler->FailWithError(err);
177 CHECK(info);
178 info->scheduled_invocations[build_file_name] = LocationRange();
179 info->all_invocations.insert(build_file_name);
180
181 if (!ScheduleBuildConfigLoadLocked(info, true, &err))
182 g_scheduler->FailWithError(err);
183 }
184
185 const Settings* ToolchainManager::GetSettingsForToolchainLocked(
186 const LocationRange& from_here,
187 const Label& toolchain_name,
188 Err* err) {
189 GetLock().AssertAcquired();
190 ToolchainMap::iterator found = toolchains_.find(toolchain_name);
191 Info* info = NULL;
192 if (found == toolchains_.end()) {
193 info = LoadNewToolchainLocked(from_here, toolchain_name, err);
194 if (!info)
195 return NULL;
196 } else {
197 info = found->second;
198 }
199 info->EnsureItemNode();
200
201 return &info->settings;
202 }
203
204 const Toolchain* ToolchainManager::GetToolchainDefinitionUnlocked(
205 const Label& toolchain_name) {
206 base::AutoLock lock(GetLock());
207 ToolchainMap::iterator found = toolchains_.find(toolchain_name);
208 if (found == toolchains_.end() || !found->second->toolchain_set)
209 return NULL;
210
211 // Since we don't allow defining a toolchain more than once, we know that
212 // once it's set it won't be mutated, so we can safely return this pointer
213 // for reading outside the lock.
214 return found->second->toolchain;
215 }
216
217 bool ToolchainManager::SetDefaultToolchainUnlocked(
218 const Label& default_toolchain,
219 const LocationRange& defined_here,
220 Err* err) {
221 base::AutoLock lock(GetLock());
222 if (!default_toolchain_.is_null()) {
223 *err = Err(defined_here, "Default toolchain already set.");
224 err->AppendSubErr(Err(default_toolchain_defined_here_,
225 "Previously defined here.",
226 "You can only set this once."));
227 return false;
228 }
229
230 if (default_toolchain.is_null()) {
231 *err = Err(defined_here, "Bad default toolchain name.",
232 "You can't set the default toolchain name to nothing.");
233 return false;
234 }
235 if (!default_toolchain.toolchain_dir().is_null() ||
236 !default_toolchain.toolchain_name().empty()) {
237 *err = Err(defined_here, "Toolchain name has toolchain.",
238 "You can't specify a toolchain (inside the parens) for a toolchain "
239 "name. I got:\n" + default_toolchain.GetUserVisibleName(true));
240 return false;
241 }
242
243 default_toolchain_ = default_toolchain;
244 default_toolchain_defined_here_ = defined_here;
245 return true;
246 }
247
248 Label ToolchainManager::GetDefaultToolchainUnlocked() const {
249 base::AutoLock lock(GetLock());
250 return default_toolchain_;
251 }
252
253 bool ToolchainManager::SetToolchainDefinitionLocked(
254 const Toolchain& tc,
255 const LocationRange& defined_from,
256 Err* err) {
257 GetLock().AssertAcquired();
258
259 ToolchainMap::iterator found = toolchains_.find(tc.label());
260 Info* info = NULL;
261 if (found == toolchains_.end()) {
262 // New toolchain.
263 info = LoadNewToolchainLocked(defined_from, tc.label(), err);
264 if (!info)
265 return false;
266 } else {
267 // It's important to preserve the exact Toolchain object in our tree since
268 // it will be in the ItemTree and targets may have dependencies on it.
269 info = found->second;
270 }
271
272 // The labels should match or else we're setting the wrong one!
273 CHECK(info->toolchain->label() == tc.label());
274
275 // Save the toolchain. We can just overwrite our definition.
276 *info->toolchain = tc;
277
278 if (info->toolchain_set) {
279 *err = Err(defined_from, "Duplicate toolchain definition.");
280 err->AppendSubErr(Err(
281 info->toolchain_definition_location,
282 "Previously defined here.",
283 "A toolchain can only be defined once. One tricky way that this could\n"
284 "happen is if your definition is itself in a file that's interpreted\n"
285 "under different toolchains, which would result in multiple\n"
286 "definitions as the file is loaded multiple times. So be sure your\n"
287 "toolchain definitions are in files that either don't define any\n"
288 "targets (probably best) or at least don't contain targets executed\n"
289 "with more than one toolchain."));
290 return false;
291 }
292
293 info->EnsureItemNode();
294
295 info->toolchain_set = true;
296 info->toolchain_definition_location = defined_from;
297 return true;
298 }
299
300 bool ToolchainManager::ScheduleInvocationLocked(
301 const LocationRange& specified_from,
302 const Label& toolchain_name,
303 const SourceDir& dir,
304 Err* err) {
305 GetLock().AssertAcquired();
306 SourceFile build_file(DirToBuildFile(dir));
307
308 // If there's no specified toolchain name, use the default.
309 ToolchainMap::iterator found;
310 if (toolchain_name.is_null())
311 found = toolchains_.find(default_toolchain_);
312 else
313 found = toolchains_.find(toolchain_name);
314
315 Info* info = NULL;
316 if (found == toolchains_.end()) {
317 // New toolchain.
318 info = LoadNewToolchainLocked(specified_from, toolchain_name, err);
319 if (!info)
320 return false;
321 } else {
322 // Use existing one.
323 info = found->second;
324 if (info->all_invocations.find(build_file) !=
325 info->all_invocations.end()) {
326 // We've already seen this source file for this toolchain, don't need
327 // to do anything.
328 return true;
329 }
330 }
331
332 info->all_invocations.insert(build_file);
333
334 switch (info->state) {
335 case TOOLCHAIN_NOT_LOADED:
336 // Toolchain needs to be loaded. Start loading it and push this buildfile
337 // on the wait queue. The actual toolchain build file will have been
338 // scheduled to be loaded by LoadNewToolchainLocked() above, so we just
339 // need to request that the build settings be loaded when that toolchain
340 // file is done executing (recall toolchain files are executed in the
341 // context of the default toolchain, which is why we need to do the extra
342 // Info lookup in the make_pair).
343 DCHECK(!default_toolchain_.is_null());
344 pending_build_config_map_[
345 std::make_pair(DirToBuildFile(toolchain_name.dir()),
346 toolchains_[default_toolchain_])] =
347 info;
348 info->scheduled_invocations[build_file] = specified_from;
349
350 // Transition to the loading state.
351 info->specified_from = specified_from;
352 info->state = TOOLCHAIN_DEFINITION_LOADING;
353 return true;
354
355 case TOOLCHAIN_DEFINITION_LOADING:
356 case TOOLCHAIN_SETTINGS_LOADING:
357 // Toolchain is in the process of loading, push this buildfile on the
358 // wait queue to run when the config is ready.
359 info->scheduled_invocations[build_file] = specified_from;
360 return true;
361
362 case TOOLCHAIN_SETTINGS_LOADED:
363 // Everything is ready, just schedule the build file to load.
364 return ScheduleBackgroundInvoke(info, specified_from, build_file, err);
365
366 default:
367 NOTREACHED();
368 return false;
369 }
370 }
371
372 // static
373 std::string ToolchainManager::ToolchainToOutputSubdir(
374 const Label& toolchain_name) {
375 // For now just assume the toolchain name is always a valid dir name. We may
376 // want to clean up the in the future.
377 return toolchain_name.name();
378 }
379
380 ToolchainManager::Info* ToolchainManager::LoadNewToolchainLocked(
381 const LocationRange& specified_from,
382 const Label& toolchain_name,
383 Err* err) {
384 GetLock().AssertAcquired();
385 Info* info = new Info(build_settings_,
386 toolchain_name,
387 ToolchainToOutputSubdir(toolchain_name));
388
389 toolchains_[toolchain_name] = info;
390
391 // Invoke the file containing the toolchain definition so that it gets
392 // defined. The default one (label is empty) will be done spearately.
393 if (!toolchain_name.is_null()) {
394 // The default toolchain should be specified whenever we're requesting
395 // another one. This is how we know under what context we should execute
396 // the invoke for the toolchain file.
397 CHECK(!default_toolchain_.is_null());
398 ScheduleInvocationLocked(specified_from, default_toolchain_,
399 toolchain_name.dir(), err);
400 }
401 return info;
402 }
403
404 void ToolchainManager::FixupDefaultToolchainLocked() {
405 // Now that we've run the default build config, we should know the
406 // default toolchain name. Fix up our reference.
407 // See Start() for more.
408 GetLock().AssertAcquired();
409 if (default_toolchain_.is_null()) {
410 g_scheduler->FailWithError(Err(Location(),
411 "Default toolchain not set.",
412 "Your build config file \"" +
413 build_settings_->build_config_file().value() +
414 "\"\ndid not call set_default_toolchain(). This is needed so "
415 "I know how to actually\ncompile your code."));
416 return;
417 }
418
419 ToolchainMap::iterator old_default = toolchains_.find(Label());
420 CHECK(old_default != toolchains_.end());
421 Info* info = old_default->second;
422 toolchains_[default_toolchain_] = info;
423 toolchains_.erase(old_default);
424
425 // Toolchain should not have been loaded in the build config file.
426 CHECK(!info->toolchain_set);
427
428 // We need to set the toolchain label now that we know it. There's no way
429 // to set the label, but we can assign the toolchain to a new one. Loading
430 // the build config can not change the toolchain, so we won't be overwriting
431 // anything useful.
432 *info->toolchain = Toolchain(&info->settings, default_toolchain_);
433 info->settings.set_is_default(true);
434 info->settings.set_toolchain_label(default_toolchain_);
435 info->EnsureItemNode();
436
437 // The default toolchain is loaded in greedy mode so all targets we
438 // encounter are generated. Non-default toolchain settings stay in non-greedy
439 // so we only generate the minimally required set.
440 info->settings.set_greedy_target_generation(true);
441
442 // Schedule a load of the toolchain build file.
443 Err err;
444 ScheduleInvocationLocked(LocationRange(), default_toolchain_,
445 default_toolchain_.dir(), &err);
446 if (err.has_error())
447 g_scheduler->FailWithError(err);
448 }
449
450 bool ToolchainManager::ScheduleBackgroundInvoke(
451 Info* info,
452 const LocationRange& specified_from,
453 const SourceFile& build_file,
454 Err* err) {
455 g_scheduler->IncrementWorkCount();
456 if (!g_scheduler->input_file_manager()->AsyncLoadFile(
457 specified_from, build_settings_, build_file,
458 base::Bind(&ToolchainManager::BackgroundInvoke,
459 base::Unretained(this), info, build_file),
460 err)) {
461 g_scheduler->DecrementWorkCount();
462 return false;
463 }
464 return true;
465 }
466
467 bool ToolchainManager::ScheduleBuildConfigLoadLocked(Info* info,
468 bool is_default,
469 Err* err) {
470 GetLock().AssertAcquired();
471
472 g_scheduler->IncrementWorkCount();
473 if (!g_scheduler->input_file_manager()->AsyncLoadFile(
474 info->specified_from, build_settings_,
475 build_settings_->build_config_file(),
476 base::Bind(&ToolchainManager::BackgroundLoadBuildConfig,
477 base::Unretained(this), info, is_default),
478 err)) {
479 g_scheduler->DecrementWorkCount();
480 return false;
481 }
482 info->state = TOOLCHAIN_SETTINGS_LOADING;
483 return true;
484 }
485
486 void ToolchainManager::BackgroundLoadBuildConfig(Info* info,
487 bool is_default,
488 const ParseNode* root) {
489 // Danger: No early returns without decrementing the work count.
490 if (root && !g_scheduler->is_failed()) {
491 // Nobody should be accessing settings at this point other than us since we
492 // haven't marked it loaded, so we can do it outside the lock.
493 Scope* base_config = info->settings.base_config();
494 base_config->set_source_dir(SourceDir("//"));
495
496 info->settings.build_settings()->build_args().SetupRootScope(
497 base_config, info->toolchain->args());
498
499 base_config->SetProcessingBuildConfig();
500 if (is_default)
501 base_config->SetProcessingDefaultBuildConfig();
502
503 ScopedTrace trace(TraceItem::TRACE_FILE_EXECUTE,
504 info->settings.build_settings()->build_config_file().value());
505 trace.SetToolchain(info->settings.toolchain_label());
506
507 const BlockNode* root_block = root->AsBlock();
508 Err err;
509 root_block->ExecuteBlockInScope(base_config, &err);
510
511 trace.Done();
512
513 base_config->ClearProcessingBuildConfig();
514 if (is_default)
515 base_config->ClearProcessingDefaultBuildConfig();
516
517 if (err.has_error()) {
518 g_scheduler->FailWithError(err);
519 } else {
520 // Base config processing succeeded.
521 Info::ScheduledInvocationMap schedule_these;
522 {
523 base::AutoLock lock(GetLock());
524 schedule_these.swap(info->scheduled_invocations);
525 info->state = TOOLCHAIN_SETTINGS_LOADED;
526 if (is_default)
527 FixupDefaultToolchainLocked();
528 }
529
530 // Schedule build files waiting on this settings. There can be many so we
531 // want to load them in parallel on the pool.
532 for (Info::ScheduledInvocationMap::iterator i = schedule_these.begin();
533 i != schedule_these.end() && !g_scheduler->is_failed(); ++i) {
534 if (!ScheduleBackgroundInvoke(info, i->second, i->first, &err)) {
535 g_scheduler->FailWithError(err);
536 break;
537 }
538 }
539 }
540 }
541 g_scheduler->DecrementWorkCount();
542 }
543
544 void ToolchainManager::BackgroundInvoke(const Info* info,
545 const SourceFile& file_name,
546 const ParseNode* root) {
547 if (root && !g_scheduler->is_failed()) {
548 if (g_scheduler->verbose_logging()) {
549 g_scheduler->Log("Running", file_name.value() + " with toolchain " +
550 info->toolchain->label().GetUserVisibleName(false));
551 }
552
553 Scope our_scope(info->settings.base_config());
554 ScopePerFileProvider per_file_provider(&our_scope);
555 our_scope.set_source_dir(file_name.GetDir());
556
557 ScopedTrace trace(TraceItem::TRACE_FILE_EXECUTE, file_name.value());
558 trace.SetToolchain(info->settings.toolchain_label());
559
560 Err err;
561 root->Execute(&our_scope, &err);
562 if (err.has_error())
563 g_scheduler->FailWithError(err);
564
565 trace.Done();
566
567 {
568 // Check to see if any build config invocations depend on this file and
569 // invoke them.
570 base::AutoLock lock(GetLock());
571 BuildConfigInvokeMap::iterator found_file =
572 pending_build_config_map_.find(std::make_pair(file_name, info));
573 if (found_file != pending_build_config_map_.end()) {
574 // The toolchain state should be waiting on the definition, which
575 // should be the thing we just loaded.
576 Info* info_to_load = found_file->second;
577 DCHECK(info_to_load->state == TOOLCHAIN_DEFINITION_LOADING);
578 DCHECK(!info_to_load->toolchain_file_loaded);
579 info_to_load->toolchain_file_loaded = true;
580
581 if (!ScheduleBuildConfigLoadLocked(info_to_load, false, &err))
582 g_scheduler->FailWithError(err);
583 pending_build_config_map_.erase(found_file);
584 }
585 }
586 }
587
588 g_scheduler->DecrementWorkCount();
589 }
590
591 base::Lock& ToolchainManager::GetLock() const {
592 return build_settings_->item_tree().lock();
593 }
OLDNEW
« no previous file with comments | « tools/gn/toolchain_manager.h ('k') | tools/gn/trace.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698