Change ExecLinkWrapper to not buffer all tool output

build/toolchain/win/tool_wrapper.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Utility functions for Windows builds.
 
 This file is copied to the build directory as part of toolchain setup and
 is used to set up calls to tools used by the build that need wrappers.
 """
 
@@ skipping 115 matching lines @@
     # https://docs.python.org/2/library/subprocess.html:
     # "On Unix with shell=True [...] if args is a sequence, the first item
     # specifies the command string, and any additional items will be treated as
     # additional arguments to the shell itself.  That is to say, Popen does the
     # equivalent of:
     #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
     # For that reason, since going through the shell doesn't seem necessary on
     # non-Windows don't do that there.
     link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    out, _ = link.communicate()
-    for line in out.splitlines():
+    # Read output one line at a time as it shows up to avoid OOM failures when
+    # GBs of output is produced.
+    while True:
+      line = link.stdout.readline()
+      if len(line) == 0: # Read of zero bytes means EOF

[scottmg, 2016/12/07 22:20:56]

I'm not sure about the loop termination...

for line in link.stdout:
  if (not ...):
    print line,
link.wait()

should be the same though and a bit nicer looking?

[brucedawson, 2016/12/07 23:00:24]

Agreed. I started with a solution that read stderr and stdout and didn't remove
the complexity when I changed it to stdout only.

+        break
       if (not line.startswith('   Creating library ') and
           not line.startswith('Generating code') and
           not line.startswith('Finished generating code')):
-        print line
+        print line,
+    link.wait()
     return link.returncode
 
   def ExecLinkWithManifests(self, arch, embed_manifest, out, ldcmd, resname,
                             mt, rc, intermediate_manifest, *manifests):
     """A wrapper for handling creating a manifest resource and then executing
     a link command."""
     # The 'normal' way to do manifests is to have link generate a manifest
     # based on gathering dependencies from the object files, then merge that
     # manifest with other manifests supplied as sources, convert the merged
     # manifest to a resource, and then *relink*, including the compiled
@@ skipping 168 matching lines @@
     project_dir = os.path.relpath(project_dir, BASE_DIR)
     selected_files = selected_files.split(';')
     ninja_targets = [os.path.join(project_dir, filename) + '^^'
         for filename in selected_files]
     cmd = ['ninja.exe']
     cmd.extend(ninja_targets)
     return subprocess.call(cmd, shell=True, cwd=BASE_DIR)
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))