supersize: Parse build.ninja concurrently

tools/binary_size/libsupersize/archive.py

 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Main Python API for analyzing binary size."""
 
 import argparse
 import calendar
 import collections
 import datetime
 import gzip
 import logging
 import os
-import multiprocessing
 import posixpath
 import re
 import subprocess
 import sys
 import tempfile
 import zipfile
 
 import describe
 import file_format
 import function_signature
@@ skipping 123 matching lines @@
 def _NormalizeSourcePath(path):
   if path.startswith('gen/'):
     # Convert gen/third_party/... -> third_party/...
     return path[4:]
   if path.startswith('../../'):
     # Convert ../../third_party/... -> third_party/...
     return path[6:]
   return path
 
 
-def _ExtractSourcePaths(symbols, output_directory):
-  """Fills in the .source_path attribute of all symbols.
-
-  Returns True if source paths were found.
-  """
-  mapper = ninja_parser.SourceFileMapper(output_directory)
-  not_found_paths = set()
+def _ExtractSourcePaths(symbols, source_mapper):
+  """Fills in the .source_path attribute of all symbols."""
+  logging.debug('Parsed %d .ninja files.', source_mapper.parsed_file_count)
 
   for symbol in symbols:
     object_path = symbol.object_path
     if symbol.source_path or not object_path:
       continue
     # We don't have source info for prebuilt .a files.
     if not os.path.isabs(object_path) and not object_path.startswith('..'):
-      source_path = mapper.FindSourceForPath(object_path)
+      source_path = source_mapper.FindSourceForPath(object_path)
       if source_path:
         symbol.source_path = _NormalizeSourcePath(source_path)
-      elif object_path not in not_found_paths:
-        not_found_paths.add(object_path)
-        logging.warning('Could not find source path for %s', object_path)
-  logging.debug('Parsed %d .ninja files.', mapper.GetParsedFileCount())
-  return len(not_found_paths) == 0
 
 
 def _CalculatePadding(symbols):
   """Populates the |padding| field based on symbol addresses.
 
   Symbols must already be sorted by |address|.
   """
   seen_sections = []
   for i, symbol in enumerate(symbols[1:]):
     prev_symbol = symbols[i]
@@ skipping 152 matching lines @@
   _CalculatePadding(size_info.raw_symbols)
   logging.info('Grouping decomposed functions')
   size_info.symbols = models.SymbolGroup(
       _ClusterSymbols(size_info.raw_symbols))
   logging.info('Processed %d symbols', len(size_info.raw_symbols))
 
 
 def CreateSizeInfo(map_path, lazy_paths=None, no_source_paths=False,
                    raw_only=False):
   """Creates a SizeInfo from the given map file."""
-  if not no_source_paths:
-    # output_directory needed for source file information.
-    lazy_paths.VerifyOutputDirectory()
   # tool_prefix needed for c++filt.
   lazy_paths.VerifyToolPrefix()
 
+  if not no_source_paths:
+    # Parse .ninja files at the same time as parsing the .map file.
+    source_mapper_result = helpers.ForkAndCall(
+        ninja_parser.Parse, lazy_paths.VerifyOutputDirectory())
+
   with _OpenMaybeGz(map_path) as map_file:
     section_sizes, raw_symbols = (
         linker_map_parser.MapFileParser().Parse(map_file))
 
   if not no_source_paths:
     logging.info('Extracting source paths from .ninja files')
-    all_found = _ExtractSourcePaths(raw_symbols, lazy_paths.output_directory)
-    assert all_found, (
+    source_mapper = source_mapper_result.get()
+    _ExtractSourcePaths(raw_symbols, source_mapper)
+    assert source_mapper.unmatched_paths_count == 0, (
         'One or more source file paths could not be found. Likely caused by '
         '.ninja files being generated at a different time than the .map file.')
 
   logging.info('Stripping linker prefixes from symbol names')
   _StripLinkerAddedSymbolPrefixes(raw_symbols)
   # Map file for some reason doesn't unmangle all names.
   # Unmangle prints its own log statement.
   _UnmangleRemainingSymbols(raw_symbols, lazy_paths.tool_prefix)
   logging.info('Normalizing object paths')
   _NormalizeObjectPaths(raw_symbols)
@@ skipping 155 matching lines @@
         models.METADATA_ELF_ARCHITECTURE: architecture,
         models.METADATA_ELF_FILENAME: relative_to_out(elf_path),
         models.METADATA_ELF_MTIME: timestamp,
         models.METADATA_ELF_BUILD_ID: build_id,
         models.METADATA_GN_ARGS: gn_args,
     }
 
     if apk_path:
       metadata[models.METADATA_APK_FILENAME] = relative_to_out(apk_path)
       # Extraction takes around 1 second, so do it in parallel.
-      pool_of_one = multiprocessing.Pool(1)
-      apk_elf_result = pool_of_one.apply_async(
-          _ElfInfoFromApk, (apk_path, apk_so_path, lazy_paths.tool_prefix))
-      pool_of_one.close()
+      apk_elf_result = helpers.ForkAndCall(
+          _ElfInfoFromApk, apk_path, apk_so_path, lazy_paths.tool_prefix)
 
   size_info = CreateSizeInfo(
       map_path, lazy_paths, no_source_paths=args.no_source_paths, raw_only=True)
 
   if metadata:
     size_info.metadata = metadata
     logging.debug('Validating section sizes')
     elf_section_sizes = _SectionSizesFromElf(elf_path, lazy_paths.tool_prefix)
     for k, v in elf_section_sizes.iteritems():
       assert v == size_info.section_sizes.get(k), (
@@ skipping 19 matching lines @@
           logging.warning('Packed section not present: %s', packed_section_name)
         else:
           size_info.section_sizes['%s (unpacked)' % packed_section_name] = (
               unstripped_section_sizes.get(packed_section_name))
 
   logging.info('Recording metadata: \n  %s',
                '\n  '.join(describe.DescribeMetadata(size_info.metadata)))
   logging.info('Saving result to %s', args.size_file)
   file_format.SaveSizeInfo(size_info, args.size_file)
   logging.info('Done')