Check in generated Dart bindings and add presubmit script

mojo/dart/tools/presubmit/check_mojom_dart.py

+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Checks that released mojom.dart files in the source tree are up to date"""
+
+import argparse
+import os
+import subprocess
+import sys
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+SRC_DIR = os.path.dirname(
+    os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(SCRIPT_DIR))))
+
+# Insert path to mojom parsing library.
+sys.path.insert(0, os.path.join(SRC_DIR,
+                                'mojo',
+                                'public',
+                                'tools',
+                                'bindings',
+                                'pylib'))
+
+from mojom.error import Error
+from mojom.parse.parser import Parse
+from mojom.parse.translate import Translate
+
+PACKAGES_DIR = os.path.join(SRC_DIR, 'mojo', 'dart', 'packages')
+
+# Script that calculates mojom output paths.
+DART_OUTPUTS_SCRIPT = os.path.join(SRC_DIR,
+                                   'mojo',
+                                   'public',
+                                   'tools',
+                                   'bindings',
+                                   'mojom_list_dart_outputs.py')
+
+# Runs command line in args from cwd. Returns the output as a string.
+def run(cwd, args):
+  return subprocess.check_output(args, cwd=cwd)
+
+
+# Given a parsed mojom, return the path of the .mojom.dart relative to its
+# package directory.
+def _mojom_output_path(mojom):
+  name = mojom['name']
+  namespace = mojom['namespace']
+  elements = ['lib']
+  elements.extend(namespace.split('.'))
+  elements.append("%s.dart" % name)
+  return os.path.join(*elements)
+
+
+# Given a parsed mojom, return the package or None.
+def _mojom_package(mojom):
+  attributes = mojom['attributes']
+  return attributes['DartPackage']
+
+
+# Load and parse a .mojom file. Returns the parsed mojom.
+def _load_mojom(path_to_mojom):
+  filename = os.path.abspath(path_to_mojom)
+  name = os.path.basename(filename)
+
+  # Read in mojom file.
+  with open(filename) as f:
+    source = f.read()
+  # Parse
+  tree = Parse(source, name)
+  mojom = Translate(tree, name)
+  return mojom
+
+
+def _print_regenerate_message(package):
+  print("""
+*** Dart Generated Bindings Check Failed for package: %s
+
+To regenerate bindings, from the src directory, run:
+./third_party/dart-sdk/dart-sdk/bin/dart mojo/dart/packages/mojom/bin/mojom.dart gen -m mojo/public -r mojo/ --output mojo/dart/packages/
+""" % (package))
+
+
+# Returns a map from package name to source directory.
+def _build_package_map():
+  packages = {}
+  for package in os.listdir(PACKAGES_DIR):
+    package_path = os.path.join(PACKAGES_DIR, package)
+    # Skip everything but directories.
+    if not os.path.isdir(package_path):
+      continue
+    packages[package] = package_path
+  return packages
+
+
+# Returns a list of paths to .mojom files vended by package_name.
+def _find_mojoms_for_package(package_name):
+  # Run git grep for all .mojom files with DartPackage="package_name"
+  try:
+    output = run(SRC_DIR, ['git',
+                           'grep',
+                           '--name-only',
+                           'DartPackage="' + package_name + '"',
+                           '--',
+                           '*.mojom'])
+  except subprocess.CalledProcessError as e:
+    # git grep exits with code 1 if nothing was found.
+    if e.returncode == 1:
+      return []
+
+  # Process output
+  mojoms = []
+  for line in output.splitlines():
+    line = line.strip()
+    # Skip empty lines.
+    if not line:
+      continue
+    mojoms.append(line)
+  return mojoms
+
+
+# Return the list of expected mojom.dart files for a package.
+def _expected_mojom_darts_for_package(mojoms):
+  output = run(SRC_DIR, ['python',
+                         DART_OUTPUTS_SCRIPT,
+                         '--mojoms'] + mojoms)
+  mojom_darts = []
+  for line in output.splitlines():
+    line = line.strip()
+    # Skip empty lines.
+    if not line:
+      continue
+    mojom_darts.append(line)
+  return mojom_darts
+
+
+# Returns a map indexed by output mojom.dart name with the value of
+# the modification time of the .mojom file in the source tree.
+def _build_expected_map(mojoms, mojom_darts):
+  assert(len(mojom_darts) == len(mojoms))
+  expected = {}
+  for i in range(0, len(mojoms)):
+    mojom_path = os.path.join(SRC_DIR, mojoms[i])
+    expected[mojom_darts[i]] = os.path.getmtime(mojom_path)
+  return expected
+
+
+# Returns a map indexed by output mojom.dart name with the value of
+# the modification time of the .mojom.dart file in the source tree.
+def _build_current_map(package):
+  current = {}
+  package_path = os.path.join(PACKAGES_DIR, package)
+  for directory, _, files in os.walk(package_path):
+    for filename in files:
+      if filename.endswith('.mojom.dart'):
+        path = os.path.abspath(os.path.join(directory, filename))
+        relpath = os.path.relpath(path, start=PACKAGES_DIR)
+        current[relpath] = os.path.getmtime(path)
+  return current
+
+
+# Checks if a mojom.dart file we expected in the source tree isn't there.
+def _check_new(package, expected, current):
+  check_failure = False
+  for mojom_dart in expected:
+    if not current.get(mojom_dart):
+      print("Package %s missing %s" % (package, mojom_dart))
+      check_failure = True
+  return check_failure
+
+
+# Checks if a mojom.dart file exists without an associated .mojom file.
+def _check_delete(package, expected, current):
+  check_failure = False
+  for mojom_dart in current:
+    if not expected.get(mojom_dart):
+      print("Package %s no longer has %s." % (package, mojom_dart))
+      print("Delete %s", os.path.join(PACKAGES_DIR, mojom_dart))
+      check_failure = True
+  return check_failure
+
+
+# Checks if a .mojom.dart file is older than the associated .mojom file.
+# TODO(johnmccutchan): Handle the case where someone edited a .mojom.dart file
+# directly instead of through the bindings generation script.
+def _check_stale(package, expected, current):
+  check_failure = False
+  for mojom_dart in expected:
+    # Missing mojom.dart file in source tree case handled by _check_new.
+    source_mtime = expected[mojom_dart]
+    if not current.get(mojom_dart):
+      continue
+    generated_mtime = current[mojom_dart]
+    if generated_mtime < source_mtime:
+      print("Package %s has old %s" % (package, mojom_dart))
+      check_failure = True
+  return check_failure
+
+
+# Returns True if any checks fail.
+def _check(package, expected, current):
+  check_failure = False
+  if _check_new(package, expected, current):
+    check_failure = True
+  if _check_stale(package, expected, current):
+    check_failure = True
+  if _check_delete(package, expected, current):
+    check_failure = True
+  return check_failure
+
+
+def global_check(packages):
+  check_failure = False
+  for package in packages:
+    mojoms = _find_mojoms_for_package(package)
+    if not mojoms:
+      continue
+    mojom_darts = _expected_mojom_darts_for_package(mojoms)
+    # We only feed in mojom files with DartPackage annotations, therefore, we
+    # should have a 1:1 mapping from mojoms[i] to mojom_darts[i].
+    assert(len(mojom_darts) == len(mojoms))
+    expected = _build_expected_map(mojoms, mojom_darts)
+    current = _build_current_map(package)
+    if _check(package, expected, current):
+      _print_regenerate_message(package)
+      check_failure = True
+  return check_failure
+
+
+def is_mojom_dart(path):
+  return path.endswith('.mojom.dart')
+
+
+def is_mojom(path):
+  return path.endswith('.mojom')
+
+
+def filter_paths(paths, path_filter):
+  result = []
+  for path in paths:
+    path = os.path.abspath(os.path.join(SRC_DIR, path))
+    if path_filter(path):
+      result.append(path)
+  return result
+
+
+def safe_mtime(path):
+  try:
+    return os.path.getmtime(path)
+  except Exception:
+    pass
+  return 0
+
+
+def presubmit_check(packages, affected_files):
+  mojoms = filter_paths(affected_files, is_mojom)
+  mojom_darts = filter_paths(affected_files, is_mojom_dart)
+  updated_mojom_dart_files = []
+  packages_with_failures = []
+  check_failure = False
+
+  # Check for updated .mojom without updated .mojom.dart
+  for mojom_file in mojoms:
+    try:
+      mojom = _load_mojom(mojom_file)
+    except Exception:
+      # Could not load .mojom file
+      return True
+
+    package = _mojom_package(mojom)
+    # If a mojom doesn't have a package, ignore it.
+    if not package:
+      continue
+    package_dir = packages.get(package)
+    # If the package isn't a known package, ignore it.
+    if not package_dir:
+      continue
+    # Expected output path relative to src.
+    mojom_dart_path = os.path.relpath(
+        os.path.join(package_dir, _mojom_output_path(mojom)), start=SRC_DIR)
+
+    mojom_mtime = safe_mtime(mojom_file)
+    mojom_dart_mtime = safe_mtime(os.path.join(SRC_DIR, mojom_dart_path))
+
+    if mojom_mtime > mojom_dart_mtime:
+      check_failure = True
+      print("Package %s has old %s" % (package, mojom_dart_path))
+      if not (package in packages_with_failures):
+        packages_with_failures.append(package)
+      continue
+    # Remember that this .mojom.dart file was updated after the .mojom file.
+    # This list is used to verify that all updated .mojom.dart files were
+    # updated because their source .mojom file changed.
+    updated_mojom_dart_files.append(mojom_dart_path)
+
+  # Check for updated .mojom.dart file without updated .mojom file.
+  for mojom_dart_file in mojom_darts:
+    # mojom_dart_file is not inside //mojo/dart/packages.
+    if not mojom_dart_file.startswith(PACKAGES_DIR):
+      continue
+
+    # Path relative to //mojo/dart/packages/
+    path_relative_to_packages = os.path.relpath(mojom_dart_file,
+                                                start=PACKAGES_DIR)
+    # Package name is first element of split path.
+    package = path_relative_to_packages.split(os.sep)[0]
+    # Path relative to src.
+    mojom_dart_path = os.path.relpath(mojom_dart_file, start=SRC_DIR)
+    # If mojom_dart_path is not in updated_mojom_dart_files, a .mojom.dart
+    # file was updated without updating the related .mojom file.
+    if not (mojom_dart_path in updated_mojom_dart_files):
+      check_failure = True
+      print("Package %s has new %s without updating source .mojom file." %
+            (package, mojom_dart_path))
+      if not (package in packages_with_failures):
+        packages_with_failures.append(package)
+
+  for package in packages_with_failures:
+    _print_regenerate_message(package)
+
+  return check_failure
+
+
+def main():
+  parser = argparse.ArgumentParser(description='Generate a dart-pkg')
+  parser.add_argument('--affected-files',
+                      action='store',
+                      metavar='affected_files',
+                      help='List of files that should be checked.',
+                      nargs='+')
+  args = parser.parse_args()
+  packages = _build_package_map()
+
+  # This script runs in two modes, the first mode is invoked by PRESUBMIT.py
+  # and passes the list of affected files. This checks for the following cases:
+  # 1) An updated .mojom file without an updated .mojom.dart file.
+  # 2) An updated .mojom.dart file without an updated .mojom file.
+  # NOTE: Case 1) also handles the case of a new .mojom file being added.
+  #
+  # The second mode does a global check of all packages under
+  # //mojo/dart/packages. This checks for the following cases:
+  # 1) An updated .mojom file without an updated .mojom.dart file.
+  # 2) A .mojom.dart file without an associated .mojom file (deletion case).
+  if args.affected_files:
+    check_failure = presubmit_check(packages, args.affected_files)
+  else:
+    check_failure = global_check(packages)
+  if check_failure:
+    return 2
+  return 0
+
+if __name__ == '__main__':
+    sys.exit(main())