Convert isolate.py to exclusively use .isolate files.

tools/isolate/merge_isolate.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Merges multiple OS-specific gyp dependency lists into one that works on all
 of them.
 
 
 The logic is relatively simple. Takes the current conditions, add more
 condition, find the strict subset. Done.
 """
 
 import copy
 import logging
 import optparse
 import re
 import sys
 
 import trace_inputs
+# Create shortcuts.
+from trace_inputs import KEY_TRACKED, KEY_UNTRACKED
 
 
 def union(lhs, rhs):
   """Merges two compatible datastructures composed of dict/list/set."""
   assert lhs is not None or rhs is not None
   if lhs is None:
     return copy.deepcopy(rhs)
   if rhs is None:
     return copy.deepcopy(lhs)
   assert type(lhs) == type(rhs), (lhs, rhs)
+  if hasattr(lhs, 'union'):
+    # Includes set, OSSettings and Configs.
+    return lhs.union(rhs)
   if isinstance(lhs, dict):
     return dict((k, union(lhs.get(k), rhs.get(k))) for k in set(lhs).union(rhs))
-  elif isinstance(lhs, set):
-    # Do not go inside the set.
-    return lhs.union(rhs)
   elif isinstance(lhs, list):
     # Do not go inside the list.
     return lhs + rhs
   assert False, type(lhs)
 
 
-def process_variables(for_os, variables):
-  """Extracts files and dirs from the |variables| dict.
-
-  Returns a list of exactly two items. Each item is a dict that maps a string
-  to a set (of strings).
-
-  In the first item, the keys are file names, and the values are sets of OS
-  names, like "win" or "mac". In the second item, the keys are directory names,
-  and the values are sets of OS names too.
-  """
-  VALID_VARIABLES = ['isolate_files', 'isolate_dirs']
-
-  # Verify strictness.
-  assert isinstance(variables, dict), variables
-  assert set(VALID_VARIABLES).issuperset(set(variables)), variables.keys()
-  for items in variables.itervalues():
-    assert isinstance(items, list), items
-    assert all(isinstance(i, basestring) for i in items), items
-
-  # Returns [files, dirs]
-  return [
-    dict((name, set([for_os])) for name in variables.get(var, []))
-    for var in VALID_VARIABLES
-  ]
-
-
 def eval_content(content):
   """Evaluates a GYP file and return the value defined in it."""
   globs = {'__builtins__': None}
   locs = {}
   value = eval(content, globs, locs)
   assert locs == {}, locs
   assert globs == {'__builtins__': None}, globs
   return value
 
 
-def _process_inner(for_os, inner, old_files, old_dirs, old_os):
-  """Processes the variables inside a condition.
-
-  Only meant to be called by parse_gyp_dict().
-
-  Args:
-  - for_os: OS where the references are tracked for.
-  - inner: Inner dictionary to process.
-  - old_files: Previous list of files to union with.
-  - old_dirs: Previous list of directories to union with.
-  - old_os: Previous list of OSes referenced to union with.
-
-  Returns:
-  - A tuple of (files, dirs, os) where each list is a union of the new
-    dependencies found for this OS, as referenced by for_os, and the previous
-    list.
+def verify_variables(variables):
+  """Verifies the |variables| dictionary is in the expected format."""
+  VALID_VARIABLES = [
+    KEY_TRACKED,
+    KEY_UNTRACKED,
+    'command',
+    'read_only',
+  ]
+  assert isinstance(variables, dict), variables
+  assert set(VALID_VARIABLES).issuperset(set(variables)), variables.keys()
+  for name, value in variables.iteritems():
+    if name == 'read_only':
+      assert value in (True, False, None), value
+    else:
+      assert isinstance(value, list), value
+      assert all(isinstance(i, basestring) for i in value), value
+
+
+def verify_condition(condition):
+  """Verifies the |condition| dictionary is in the expected format."""
+  VALID_INSIDE_CONDITION = ['variables']
+  assert isinstance(condition, list), condition
+  assert 2 <= len(condition) <= 3, condition
+  assert re.match(r'OS==\"([a-z]+)\"', condition[0]), condition[0]
+  for c in condition[1:]:
+    assert isinstance(c, dict), c
+    assert set(VALID_INSIDE_CONDITION).issuperset(set(c)), c.keys()
+    verify_variables(c.get('variables', {}))
+
+
+def verify_root(value):
+  VALID_ROOTS = ['variables', 'conditions']
+  assert isinstance(value, dict), value
+  assert set(VALID_ROOTS).issuperset(set(value)), value.keys()
+  verify_variables(value.get('variables', {}))
+
+  conditions = value.get('conditions', [])
+  assert isinstance(conditions, list), conditions
+  for condition in conditions:
+    verify_condition(condition)
+
+
+class OSSettings(object):
+  """Represents the dependencies for an OS. The structure is immutable."""
+  def __init__(self, name, values):
+    self.name = name
+    verify_variables(values)
+    self.tracked = sorted(values.get(KEY_TRACKED, []))
+    self.untracked = sorted(values.get(KEY_UNTRACKED, []))
+    self.command = values.get('command', [])[:]
+    self.read_only = values.get('read_only')
+
+  def union(self, rhs):
+    assert self.name == rhs.name
+    assert not (self.command and rhs.command)
+    var = {
+      KEY_TRACKED: sorted(self.tracked + rhs.tracked),
+      KEY_UNTRACKED: sorted(self.untracked + rhs.untracked),
+      'command': self.command or rhs.command,
+      'read_only': rhs.read_only if self.read_only is None else self.read_only,
+    }
+    return OSSettings(self.name, var)
+
+  def flatten(self):
+    out = {}
+    if self.command:
+      out['command'] = self.command
+    if self.tracked:
+      out[KEY_TRACKED] = self.tracked
+    if self.untracked:
+      out[KEY_UNTRACKED] = self.untracked
+    if self.read_only is not None:
+      out['read_only'] = self.read_only
+    return out
+
+
+class Configs(object):
+  """Represents all the OS-specific configurations.
+
+  The self.per_os[None] member contains all the 'else' clauses plus the default
+  values. It is not included in the flatten() result.
   """
-  assert isinstance(inner, dict), inner
-  assert set(['variables']).issuperset(set(inner)), inner.keys()
-  new_files, new_dirs = process_variables(for_os, inner.get('variables', {}))
-  if new_files or new_dirs:
-    old_os = old_os.union([for_os.lstrip('!')])
-  return union(old_files, new_files), union(old_dirs, new_dirs), old_os
-
-
-def parse_gyp_dict(value):
-  """Parses a gyp dict as returned by eval_content().
+  def __init__(self, oses):
+    self.per_os = {
+        None: OSSettings(None, {}),
+    }
+    self.per_os.update(dict((name, OSSettings(name, {})) for name in oses))
+
+  def union(self, rhs):
+    out = Configs(list(set(self.per_os.keys() + rhs.per_os.keys())))
+    for value in self.per_os.itervalues():
+      # TODO(maruel): FAIL
+      out = out.union(value)
+    for value in rhs.per_os.itervalues():
+      out = out.union(value)
+    return out
+
+  def add_globals(self, values):
+    for key in self.per_os:
+      self.per_os[key] = self.per_os[key].union(OSSettings(key, values))
+
+  def add_values(self, for_os, values):
+    self.per_os[for_os] = self.per_os[for_os].union(OSSettings(for_os, values))
+
+  def add_negative_values(self, for_os, values):
+    """Includes the variables to all OSes except |for_os|.
+
+    This includes 'None' so unknown OSes gets it too.
+    """
+    for key in self.per_os:
+      if key != for_os:
+        self.per_os[key] = self.per_os[key].union(OSSettings(key, values))
+
+  def flatten(self):
+    """Returns a flat dictionary representation of the configuration.
+
+    Skips None pseudo-OS.
+    """
+    return dict(
+        (k, v.flatten()) for k, v in self.per_os.iteritems() if k is not None)
+
+
+def invert_map(variables):
+  """Converts a dict(OS, dict(deptype, list(dependencies)) to
+  dict(deptype, dict(dependency, set(OSes)) for easier processing.

[Roger Tawa OOO till Jul 10th, 2012/04/11 15:15:03]

also returns 'OSes' key in dict.  maybe the return value should be a tuple:
dict, set(OSes)

+  """
+  KEYS = (
+    KEY_TRACKED,
+    KEY_UNTRACKED,
+    'command',
+    'read_only',
+  )
+  out = dict((key, {}) for key in KEYS)
+  out['OSes'] = set(variables)
+  for os_name, values in variables.iteritems():
+    for key in (KEY_TRACKED, KEY_UNTRACKED):
+      for item in values.get(key, []):
+        out[key].setdefault(item, set()).add(os_name)
+
+    # command needs special handling.
+    command = tuple(values.get('command', []))
+    out['command'].setdefault(command, set()).add(os_name)
+
+    # read_only needs special handling.
+    out['read_only'].setdefault(values.get('read_only'), set()).add(os_name)
+  return out
+
+
+def reduce_inputs(values):
+  """Reduces the invert_map() output to the strictest minimum list."""
+  # Construct the inverse map first.
+  # Look at each individual file and directory, map where they are used and
+  # reconstruct the inverse dictionary.
+  # Do not convert back to negative if only 2 OSes were merged.
+  KEYS = (
+    KEY_TRACKED,
+    KEY_UNTRACKED,
+    'command',
+    'read_only',
+  )
+  out = dict((key, {}) for key in KEYS)
+  oses = out['OSes'] = set(values['OSes'])
+  assert None not in oses, oses
+  if len(oses) > 2:
+    for key in KEYS:
+      for item, item_oses in values.get(key, {}).iteritems():
+        # Converts all oses.difference('foo') to '!foo'.
+        assert None not in item_oses, item_oses
+        missing = oses.difference(item_oses)
+        if len(missing) == 1:
+          # Replace it with a negative.
+          out[key][item] = set(['!' + tuple(missing)[0]])
+        elif not missing:
+          out[key][item] = set([None])
+        else:
+          out[key][item] = set(item_oses)
+  return out
+
+
+def convert_map_to_gyp(values):
+  """Regenerates back a gyp-like configuration dict from files and dirs
+  mappings generated from reduce_inputs()."""
+  # First, inverse the mapping to make it dict first.
+  oses = values['OSes']
+  config = {}
+  for key in values:
+    if key == 'OSes':
+      continue
+    for item, oses in values[key].iteritems():
+      if item is None:
+        # For read_only default.
+        continue
+      for cond_os in oses:
+        cond_key = None if cond_os is None else cond_os.lstrip('!')
+        # Insert the if/else dicts.
+        condition_values = config.setdefault(cond_key, [{}, {}])
+        # If condition is negative, use index 1, else use index 0.
+        cond_value = condition_values[int((cond_os or '').startswith('!'))]
+        variables = cond_value.setdefault('variables', {})
+
+        if item in (True, False):
+          # One-off for read_only.
+          variables[key] = item
+        else:
+          if isinstance(item, tuple):
+            # One-off for command.
+            # Do not merge lists and do not sort!
+            # Note that item is a tuple.
+            assert key not in variables
+            variables[key] = list(item)
+          else:
+            # The list of items (files or dirs). Append the new item and keep
+            # the list sorted.
+            l = variables.setdefault(key, [])
+            l.append(item)
+            l.sort()
+
+  out = {}
+  for o in sorted(config):
+    d = config[o]
+    if o is None:
+      assert not d[1]
+      out = union(out, d[0])
+    else:
+      c = out.setdefault('conditions', [])
+      if d[1]:
+        c.append(['OS=="%s"' % o] + d)
+      else:
+        c.append(['OS=="%s"' % o] + d[0:1])
+  return out
+
+
+def load_gyp(value):
+  """Parses one gyp skeleton and returns a Configs() instance.
 
   |value| is the loaded dictionary that was defined in the gyp file.
 
-  Returns a 3-tuple, where the first two items are the same as the items
-  returned by process_variable() in the same order, and the last item is a set
-  of strings of all OSs seen in the input dict.
-
   The expected format is strict, anything diverting from the format below will
-  fail:
+  throw an assert:
   {
     'variables': {
-      'isolate_files': [
+      'command': [
         ...
       ],
-      'isolate_dirs: [
+      'isolate_dependency_tracked': [
         ...
       ],
+      'isolate_dependency_untracked': [
+        ...
+      ],
+      'read_only': False,
     },
     'conditions': [
       ['OS=="<os>"', {
         'variables': {
           ...
         },
       }, {  # else
         'variables': {
           ...
         },
       }],
       ...
     ],
   }
   """
-  assert isinstance(value, dict), value
-  VALID_ROOTS = ['variables', 'conditions']
-  assert set(VALID_ROOTS).issuperset(set(value)), value.keys()
+  verify_root(value)
+
+  # Scan to get the list of OSes.
+  conditions = value.get('conditions', [])
+  oses = set(re.match(r'OS==\"([a-z]+)\"', c[0]).group(1) for c in conditions)
+  configs = Configs(oses)
 
   # Global level variables.
-  oses = set()
-  files, dirs = process_variables(None, value.get('variables', {}))
+  configs.add_globals(value.get('variables', {}))
 
   # OS specific variables.
-  conditions = value.get('conditions', [])
-  assert isinstance(conditions, list), conditions
   for condition in conditions:
-    assert isinstance(condition, list), condition
-    assert 2 <= len(condition) <= 3, condition
-    m = re.match(r'OS==\"([a-z]+)\"', condition[0])
-    assert m, condition[0]
-    condition_os = m.group(1)
-
-    files, dirs, oses = _process_inner(
-        condition_os, condition[1], files, dirs, oses)
-
-    if len(condition) == 3:
-      files, dirs, oses = _process_inner(
-          '!' + condition_os, condition[2], files, dirs, oses)
-
-  # TODO(maruel): _expand_negative() should be called here, because otherwise
-  # the OSes the negative condition represents is lost once the gyps are merged.
-  # This cause an invalid expansion in reduce_inputs() call.
-  return files, dirs, oses
+    condition_os = re.match(r'OS==\"([a-z]+)\"', condition[0]).group(1)
+    configs.add_values(condition_os, condition[1].get('variables', {}))
+    if len(condition) > 2:
+      configs.add_negative_values(
+          condition_os, condition[2].get('variables', {}))
+  return configs
 
 
-def parse_gyp_dicts(gyps):
+def load_gyps(items):
   """Parses each gyp file and returns the merged results.
 
-  It only loads what parse_gyp_dict() can process.
+  It only loads what load_gyp() can process.
 
   Return values:
     files: dict(filename, set(OS where this filename is a dependency))
     dirs:  dict(dirame, set(OS where this dirname is a dependency))
     oses:  set(all the OSes referenced)
     """
-  files = {}
-  dirs = {}
-  oses = set()
-  for gyp in gyps:
-    with open(gyp, 'rb') as gyp_file:
-      content = gyp_file.read()
-    gyp_files, gyp_dirs, gyp_oses = parse_gyp_dict(eval_content(content))
-    files = union(gyp_files, files)
-    dirs = union(gyp_dirs, dirs)
-    oses |= gyp_oses
-  return files, dirs, oses
+  configs = Configs([])
+  for item in items:
+    configs = configs.union(load_gyp(eval_content(open(item, 'rb').read())))
+  return configs
 
 
-def _expand_negative(items, oses):
-  """Converts all '!foo' value in the set by oses.difference('foo')."""
-  assert None not in oses and len(oses) >= 2, oses
-  for name in items:
-    if None in items[name]:
-      # Shortcut any item having None in their set. An item listed in None means
-      # the item is a dependency on all OSes. As such, there is no need to list
-      # any OS.
-      items[name] = set([None])
-      continue
-    for neg in [o for o in items[name] if o.startswith('!')]:
-      # Replace it with the inverse.
-      items[name] = items[name].union(oses.difference([neg[1:]]))
-      items[name].remove(neg)
-    if items[name] == oses:
-      items[name] = set([None])
-
-
-def _compact_negative(items, oses):
-  """Converts all oses.difference('foo') to '!foo'.
-
-  It is doing the reverse of _expand_negative().
-  """
-  assert None not in oses and len(oses) >= 3, oses
-  for name in items:
-    missing = oses.difference(items[name])
-    if len(missing) == 1:
-      # Replace it with a negative.
-      items[name] = set(['!' + tuple(missing)[0]])
-
-
-def reduce_inputs(files, dirs, oses):
-  """Reduces the variables to their strictest minimum."""
-  # Construct the inverse map first.
-  # Look at each individual file and directory, map where they are used and
-  # reconstruct the inverse dictionary.
-  # First, expands all '!' builders into the reverse.
-  # TODO(maruel): This is too late to call _expand_negative(). The exact list
-  # negative OSes condition it represents is lost at that point.
-  _expand_negative(files, oses)
-  _expand_negative(dirs, oses)
-
-  # Do not convert back to negative if only 2 OSes were merged. It is easier to
-  # read this way.
-  if len(oses) > 2:
-    _compact_negative(files, oses)
-    _compact_negative(dirs, oses)
-
-  return files, dirs
-
-
-def convert_to_gyp(files, dirs):
-  """Regenerates back a gyp-like configuration dict from files and dirs
-  mappings.
-
-  Sort the lists.
-  """
-  # First, inverse the mapping to make it dict first.
-  config = {}
-  def to_cond(items, name):
-    for item, oses in items.iteritems():
-      for cond_os in oses:
-        condition_values = config.setdefault(
-            None if cond_os is None else cond_os.lstrip('!'),
-            [{}, {}])
-        # If condition is negative, use index 1, else use index 0.
-        condition_value = condition_values[int((cond_os or '').startswith('!'))]
-        # The list of items (files or dirs). Append the new item and keep the
-        # list sorted.
-        l = condition_value.setdefault('variables', {}).setdefault(name, [])
-        l.append(item)
-        l.sort()
-
-  to_cond(files, 'isolate_files')
-  to_cond(dirs, 'isolate_dirs')
-
-  out = {}
-  for o in sorted(config):
-    d = config[o]
-    if o is None:
-      assert not d[1]
-      out = union(out, d[0])
-    else:
-      c = out.setdefault('conditions', [])
-      if d[1]:
-        c.append(['OS=="%s"' % o] + d)
-      else:
-        c.append(['OS=="%s"' % o] + d[0:1])
-  return out
-
-
-def main():
+def main(args=None):
   parser = optparse.OptionParser(
       usage='%prog <options> [file1] [file2] ...')
   parser.add_option(
       '-v', '--verbose', action='count', default=0, help='Use multiple times')
 
-  options, args = parser.parse_args()
+  options, args = parser.parse_args(args)
   level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)]
   logging.basicConfig(
         level=level,
         format='%(levelname)5s %(module)15s(%(lineno)3d):%(message)s')
 
   trace_inputs.pretty_print(
-      convert_to_gyp(*reduce_inputs(*parse_gyp_dicts(args))),
+      convert_map_to_gyp(reduce_inputs(invert_map(load_gyps(args).flatten()))),
       sys.stdout)
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())