Verify that gclient solution looks correct.

tools/check_git_config.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Script that attempts to push to a special git repository to verify that git
-credentials are configured correctly. It also attempts to fix misconfigurations
-if possible.
+credentials are configured correctly. It also verifies that gclient solution is
+configured to use git checkout.
 
 It will be added as gclient hook shortly before Chromium switches to git and
 removed after the switch.
 
 When running as hook in *.corp.google.com network it will also report status
 of the push attempt to the server (on appengine), so that chrome-infra team can
-collect information about misconfigured Git accounts (to fix them).
-
-When invoked manually will do the access test and submit the report regardless
-of where it is running.
+collect information about misconfigured Git accounts.
 """
 
 import contextlib
+import datetime
 import errno
 import getpass
 import json
 import logging
 import netrc
 import optparse
 import os
+import pprint
 import shutil
 import socket
 import ssl
 import subprocess
 import sys
 import tempfile
 import time
 import urllib2
 import urlparse
 
 
 # Absolute path to src/ directory.
 REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+# Absolute path to a file with gclient solutions.
+GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
+
 # Incremented whenever some changes to scrip logic are made. Change in version
 # will cause the check to be rerun on next gclient runhooks invocation.
 CHECKER_VERSION = 0
 
+# Do not attempt to upload a report after this date.
+UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1)
+
 # URL to POST json with results to.
 MOTHERSHIP_URL = (
     'https://chromium-git-access.appspot.com/'
     'git_access/api/v1/reports/access_check')
 
 # Repository to push test commits to.
 TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test'
 
+# Git-compatible gclient solution.
+GOOD_GCLIENT_SOLUTION = {
+  'name': 'src',
+  'deps_file': '.DEPS.git',
+  'managed': False,
+  'url': 'https://chromium.googlesource.com/chromium/src.git',
+}
+
 # Possible chunks of git push response in case .netrc is misconfigured.
 BAD_ACL_ERRORS = (
   '(prohibited by Gerrit)',
   'does not match your user account',
   'Invalid user name or password',
 )
 
 
 def is_on_bot():
   """True when running under buildbot."""
@@ skipping 53 matching lines @@
   try:
     proc = subprocess.Popen(['git', '--version'], stdout=subprocess.PIPE)
     out, _ = proc.communicate()
     return out.strip() if proc.returncode == 0 else ''
   except OSError as exc:
     if exc.errno != errno.ENOENT:
       logging.exception('Unexpected error when calling git')
     return ''
 
 
+def read_gclient_solution():
+  """Read information about 'src' gclient solution from .gclient file.
+
+  Returns tuple:
+    (url, deps_file, managed)
+    or
+    (None, None, None) if no such solution.
+  """
+  try:
+    env = {}
+    execfile(GCLIENT_CONFIG, env, env)
+    for sol in env['solutions']:
+      if sol['name'] == 'src':
+        return sol.get('url'), sol.get('deps_file'), sol.get('managed')

[Vadim Sh., 2014/08/18 23:29:21]

I intentionally skip custom_deps and other stuff. I want to include
gclient-related data into the report, and I want to keep the report flat
key-value map. Simplest approach is to preselect only keys we are interested in.

+    return None, None, None
+  except Exception:
+    logging.exception('Failed to read .gclient solution')
+    return None, None, None
+
+
 def scan_configuration():
   """Scans local environment for git related configuration values."""
   # Git checkout?
   is_git = is_using_git()
 
   # On Windows HOME should be set.
   if 'HOME' in os.environ:
     netrc_path = os.path.join(
         os.environ['HOME'],
         '_netrc' if sys.platform.startswith('win') else '.netrc')
   else:
     netrc_path = None
 
   # Netrc exists?
   is_using_netrc = netrc_path and os.path.exists(netrc_path)
 
   # Read it.
   netrc_obj = None
   if is_using_netrc:
     try:
       netrc_obj = netrc.netrc(netrc_path)
     except Exception:
       logging.exception('Failed to read netrc from %s', netrc_path)
       netrc_obj = None
 
+  # Read gclient 'src' solution.
+  gclient_url, gclient_deps, gclient_managed = read_gclient_solution()
+
   return {
     'checker_version': CHECKER_VERSION,
     'is_git': is_git,
     'is_home_set': 'HOME' in os.environ,
     'is_using_netrc': is_using_netrc,
     'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0,
     'git_version': get_git_version(),
     'platform': sys.platform,
     'username': getpass.getuser(),
     'git_user_email': read_git_config('user.email') if is_git else '',
     'git_user_name': read_git_config('user.name') if is_git else '',
     'chromium_netrc_email':
         read_netrc_user(netrc_obj, 'chromium.googlesource.com'),
     'chrome_internal_netrc_email':
         read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'),
+    'gclient_deps': gclient_deps,
+    'gclient_managed': gclient_managed,
+    'gclient_url': gclient_url,
   }
 
 
 def last_configuration_path():
   """Path to store last checked configuration."""
   if is_using_git():
     return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json')
   elif is_using_svn():
     return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json')
   else:
@@ skipping 58 matching lines @@
     self.append_to_log(out)
     return retcode
 
   def append_to_log(self, text):
     if text:
       self.log.append(text)
       if self.verbose:
         logging.warning(text)
 
 
-def check_git_access(conf, report_url, verbose):
+def check_git_config(conf, report_url, verbose):
   """Attempts to push to a git repository, reports results to a server.
 
   Returns True if the check finished without incidents (push itself may
   have failed) and should NOT be retried on next invocation of the hook.
   """
-  logging.warning('Checking push access to the git repository...')
-
   # Don't even try to push if netrc is not configured.
   if not conf['chromium_netrc_email']:
     return upload_report(
         conf,
         report_url,
         verbose,
         push_works=False,
         push_log='',
         push_duration_ms=0)
 
   # Ref to push to, each user has its own ref.
   ref = 'refs/push-test/%s' % conf['chromium_netrc_email']
 
   push_works = False
   flake = False
   started = time.time()
   try:
+    logging.warning('Checking push access to the git repository...')
     with temp_directory() as tmp:
       # Prepare a simple commit on a new timeline.
       runner = Runner(tmp, verbose)
       runner.run(['git', 'init', '.'])
       if conf['git_user_name']:
         runner.run(['git', 'config', 'user.name', conf['git_user_name']])
       if conf['git_user_email']:
         runner.run(['git', 'config', 'user.email', conf['git_user_email']])
       with open(os.path.join(tmp, 'timestamp'), 'w') as f:
         f.write(str(int(time.time() * 1000)))
       runner.run(['git', 'add', 'timestamp'])
       runner.run(['git', 'commit', '-m', 'Push test.'])
       # Try to push multiple times if it fails due to issues other than ACLs.
       attempt = 0
       while attempt < 5:
         attempt += 1
         logging.info('Pushing to %s %s', TEST_REPO_URL, ref)
         ret = runner.run(['git', 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f'])
         if not ret:
           push_works = True
           break
         if any(x in runner.log[-1] for x in BAD_ACL_ERRORS):
           push_works = False
           break
   except Exception:
     logging.exception('Unexpected exception when pushing')
     flake = True
 
+  if push_works:
+    logging.warning('Git push works!')
+  else:
+    logging.warning(
+        'Git push doesn\'t work, which is fine if you are not a committer.')
+
   uploaded = upload_report(
       conf,
       report_url,
       verbose,
       push_works=push_works,
       push_log='\n'.join(runner.log),
       push_duration_ms=int((time.time() - started) * 1000))
   return uploaded and not flake
 
 
+def check_gclient_config(conf):
+  """Shows warning if gclient solution is not properly configured for git."""
+  current = {
+    'name': 'src',
+    'deps_file': conf['gclient_deps'],
+    'managed': conf['gclient_managed'],
+    'url': conf['gclient_url'],
+  }
+  if current != GOOD_GCLIENT_SOLUTION:
+    print '-' * 80
+    print 'Your gclient solution is not set to use supported git workflow!'
+    print
+    print 'Your \'src\' solution (in %s):' % GCLIENT_CONFIG
+    print pprint.pformat(current, indent=2)
+    print
+    print 'Correct \'src\' solution to use git:'
+    print pprint.pformat(GOOD_GCLIENT_SOLUTION, indent=2)
+    print
+    print 'Please update your .gclient file ASAP.'
+    print '-' * 80
+
+
 def upload_report(
     conf, report_url, verbose, push_works, push_log, push_duration_ms):
   """Posts report to the server, returns True if server accepted it.
 
   Uploads the report only if script is running in Google corp network. Otherwise
   just prints the report.
   """
   report = conf.copy()
   report.update(
       push_works=push_works,
       push_log=push_log,
       push_duration_ms=push_duration_ms)
 
   as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True)
-
-  if push_works:
-    logging.warning('Git push works!')
-  else:
-    logging.warning(
-        'Git push doesn\'t work, which is fine if you are not a committer.')
-
   if verbose:
     print 'Status of git push attempt:'
     print as_bytes
 
-  # Do not upload it outside of corp.
-  if not is_in_google_corp():
+  # Do not upload it outside of corp or if server side is already disabled.
+  if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS:
     if verbose:
       print (
           'You can send the above report to chrome-git-migration@google.com '
           'if you need help to set up you committer git account.')
     return True
 
   req = urllib2.Request(
       url=report_url,
       data=as_bytes,
       headers={'Content-Type': 'application/json; charset=utf-8'})
@@ skipping 33 matching lines @@
       '--verbose',
       action='store_true',
       help='More logging')
   options, args = parser.parse_args()
   if args:
     parser.error('Unknown argument %s' % args)
   logging.basicConfig(
       format='%(message)s',
       level=logging.INFO if options.verbose else logging.WARN)
 
-  # When invoked not as hook, always run the check.
+  # When invoked not as a hook, always run the check.
   if not options.running_as_hook:
-    if check_git_access(scan_configuration(), options.report_url, True):
-      return 0
-    return 1
-
-  # Otherwise, do it only on google owned, non-bot machines.
-  if is_on_bot() or not is_in_google_corp():
-    logging.info('Skipping the check: bot or non corp.')
+    config = scan_configuration()
+    check_gclient_config(config)
+    check_git_config(config, options.report_url, True)
     return 0
 
-  # Skip the check if current configuration was already checked.
+  # Always do nothing on bots.
+  if is_on_bot():
+    return 0
+
+  # Read current config, verify gclient solution looks correct.
   config = scan_configuration()
+  check_gclient_config(config)
+
+  # Do not attempt to push from non-google owned machines.
+  if not is_in_google_corp():
+    logging.info('Skipping git push check: non *.corp.google.com machine.')
+    return 0
+
+  # Skip git push check if current configuration was already checked.
   if config == read_last_configuration():
     logging.info('Check already performed, skipping.')
     return 0
 
   # Run the check. Mark configuration as checked only on success. Ignore any
   # exceptions or errors. This check must not break gclient runhooks.
   try:
-    ok = check_git_access(config, options.report_url, False)
+    ok = check_git_config(config, options.report_url, False)
     if ok:
       write_last_configuration(config)
     else:
       logging.warning('Check failed and will be retried on the next run')
   except Exception:
     logging.exception('Unexpected exception when performing git access check')
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))