Enable whitelist generation for all builds.

build/toolchain/wrapper_utils.py

Index: build/toolchain/wrapper_utils.py
diff --git a/build/toolchain/wrapper_utils.py b/build/toolchain/wrapper_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac7819864617938d86ddff6936f4cd66fe123f3c
--- /dev/null
+++ b/build/toolchain/wrapper_utils.py
@@ -0,0 +1,132 @@
+# Copyright (c) 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helper functions for gcc_toolchain.gni wrappers."""
+
+import os
+import re
+import subprocess
+import sys
+
+PRAGMA_WARNING = '[-Wunknown-pragmas]'

[agrieve, 2016/07/27 01:42:27]

nit: Prefix with _

[estevenson, 2016/07/28 21:56:46]

Done.

+PRAGMA_WARNING_LINE_COUNT = 9
+BAT_PREFIX = 'cmd /c call '
+
+
+def CommandToRun(command):
+  """Generates commands compatible with Windows.
+
+  When running on a Windows host and using a toolchain whose tools are
+  actually wrapper scripts (i.e. .bat files on Windows) rather than binary
+  executables, the |command| to run has to be prefixed with this magic.
+  The GN toolchain definitions take care of that for when GN/Ninja is
+  running the tool directly.  When that command is passed in to this
+  script, it appears as a unitary string but needs to be split up so that
+  just 'cmd' is the actual command given to Python's subprocess module.
+
+  Args:
+    command: List containing the UNIX style |command|.
+
+  Returns:
+    A list containing the Windows version of the |command|.
+  """
+  if command[0].startswith(BAT_PREFIX):
+    command = command[0].split(None, 3) + command[1:]
+  return command
+
+
+def CombineWhitelists(rspfile, outfile):

[agrieve, 2016/07/27 01:42:28]

nit: CombineWhitelists -> CombineResourceWhitelists

[estevenson, 2016/07/28 21:56:45]

Done.

+  """Combines all whitelists for a resource file into a single whitelist.
+
+  Args:
+    rspfile: Path to the resource file containing all targets.
+    outfile: Path to save the combined whitelist.
+  """
+  with open(rspfile, 'r') as f:
+    tokens = f.read().split()

[agrieve, 2016/07/27 01:42:27]

nit: split won't properly handle filenames with spaces in them. Should use
shlex.split instead.

[estevenson, 2016/07/28 21:56:45]

Done.

+    whitelists = ['%s.whitelist' % token for token in tokens

[agrieve, 2016/07/27 01:42:27]

nit: unindent

[estevenson, 2016/07/28 21:56:46]

Done.

+                  if os.path.exists('%s.whitelist' % token)]
+
+  resources = set()
+  for whitelist in whitelists:
+    with open(whitelist, 'r') as f:
+      resources.update(f.readlines())
+
+  with open(outfile, 'w') as f:
+    f.writelines(resources)
+
+
+def GetResourceIdsInPragmaWarnings(text):

[agrieve, 2016/07/27 01:42:27]

nit: never prefix a non-trivial function with "Get". In this case, maybe go for
"Extract"

[estevenson, 2016/07/28 21:56:46]

Done.

+  """Returns set of resource IDs that are inside unknown pragma warnings.
+
+  Args:
+    text: The text that will be scanned for unknown pragma warnings.
+
+  Returns:
+    A set containing integers representing resource IDs.
+  """
+  used_resources = set()
+  lines = text.splitlines()
+  unknown_pragma_warning_pattern = re.compile(

[agrieve, 2016/07/27 01:42:27]

nit: Make this a constant at the top of the file.

[estevenson, 2016/07/28 21:56:46]

Done.

+      'whitelisted_resource_(?P<resource_id>[0-9]+)')
+  for ln in lines:
+    match = unknown_pragma_warning_pattern.search(ln)
+    if match:
+      resource_id = int(match.group('resource_id'))
+      used_resources.add(resource_id)
+
+  return used_resources
+
+
+def _FilterStderr(text, warning, warning_size):
+  """Removes warnings from compilation output.
+
+  Args:
+    text: A String containing warnings.
+    warning: A string that identifies the warning type to surpress.
+    warning_size: The number of lines a warning corresponding to the given
+      |warning| occupies.
+
+  Returns:
+    A string containing the filtered text.
+  """
+  # TODO(estevenson): Find a cleaner, less error prone way to do this.

[agrieve, 2016/07/27 01:42:27]

Yep, ideally, this should be combined with the above function since they both
need to extract the warnings.

Also - need to be able to support both GCC and clang. This function is probably
worth of having a unittests that includes example compiler output.

[estevenson, 2016/07/28 21:56:46]

We discussed this, there doesn't seem to be a clean/reliable way to do this so
we will leave the output in for now.

+  lines = text.splitlines()
+  warning_groups = [range(lineno, lineno + warning_size)
+                    for lineno, line in enumerate(lines)
+                    if warning in line]
+  warning_lines = [line for group in warning_groups for line in group]
+  filtered_lines = [line for i, line in enumerate(lines)
+                    if i not in warning_lines]
+  filtered_text = '\n'.join(line for line in filtered_lines)
+  return filtered_text
+
+
+def FilterUnknownPragmas(stderr):

[agrieve, 2016/07/27 01:42:27]

Note: We only want to filter warnings that are cause by whitelist pragmas. If
there were another unknown pragma, we'd want to print it.

[estevenson, 2016/07/28 21:56:45]

Done.

+  """Removes unknown pragma warnings from stderr.
+
+  Args:
+    stderr: Stderr output captured from a compilation command.
+
+  Returns:
+    A string containing the filtered stderr.
+  """
+  return _FilterStderr(stderr, PRAGMA_WARNING, PRAGMA_WARNING_LINE_COUNT)
+
+
+def GetCommandOutput(args, cwd=None, env=None):

[agrieve, 2016/07/27 01:42:28]

nit: remove optional params since they are not used.

[estevenson, 2016/07/28 21:56:46]

Done.

+  """Returns the output of a command.
+
+  Args:
+    args: A list containing the command and arguments.
+    cwd: The working directory from where the command should be made.
+    env: Environment variables for the new process.
+  """
+  if not cwd:
+    cwd = os.getcwd()
+
+  child = subprocess.Popen(args,
+      stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)

[agrieve, 2016/07/27 01:42:28]

There's no need to pipe stdout since we're not filtering it. I think omitting it
causes the parent stdout to be inherited, so it should "just work"

[estevenson, 2016/07/28 21:56:46]

Done.

+  stdout, stderr = child.communicate()
+  return child.returncode, stdout, stderr