[gms updater] Fixes and tests to prepare activation

build/android/play_services/update.py

Index: build/android/play_services/update.py
diff --git a/build/android/play_services/update.py b/build/android/play_services/update.py
index b2d57ad359f49c7c271246d4f0b629b332b551d0..c6c8dcc1f13847e85b83f7e86df1049c17dd2f3e 100755
--- a/build/android/play_services/update.py
+++ b/build/android/play_services/update.py
@@ -4,12 +4,11 @@
 # found in the LICENSE file.
 
 '''
-Script to help uploading and downloading the Google Play services client
-library to and from a Google Cloud storage.
+Script to help uploading and downloading the Google Play services library to
+and from a Google Cloud storage.
 '''
 
 import argparse
-import collections
 import logging
 import os
 import re
@@ -37,7 +36,7 @@ SHA1_DIRECTORY = os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'android',
                               'play_services')
 
 # Default bucket used for storing the files.
-GMS_CLOUD_STORAGE = 'chrome-sdk-extras'
+GMS_CLOUD_STORAGE = 'chromium-android-tools/play-services'
 
 # Path to the default configuration file. It exposes the currently installed
 # version of the library in a human readable way.
@@ -45,13 +44,13 @@ CONFIG_DEFAULT_PATH = os.path.join(constants.DIR_SOURCE_ROOT, 'build',
                                    'android', 'play_services', 'config.json')
 
 LICENSE_FILE_NAME = 'LICENSE'
-LIBRARY_FILE_NAME = 'google_play_services_library.zip'
+ZIP_FILE_NAME = 'google_play_services_library.zip'
 GMS_PACKAGE_ID = 'extra-google-google_play_services'  # used by sdk manager
 
 LICENSE_PATTERN = re.compile(r'^Pkg\.License=(?P<text>.*)$', re.MULTILINE)
 
 
-def Main():
+def main(raw_args):
   parser = argparse.ArgumentParser(
       description=__doc__ + 'Please see the subcommand help for more details.',
       formatter_class=utils.DefaultsRawHelpFormatter)
@@ -63,27 +62,18 @@ def Main():
       help='download the library from the cloud storage',
       description=Download.__doc__,
       formatter_class=utils.DefaultsRawHelpFormatter)
-  parser_download.add_argument('-f', '--force',
-                               action='store_true',
-                               help=('run even if the local version is '
-                                     'already up to date'))
   parser_download.set_defaults(func=Download)
-  AddCommonArguments(parser_download)
+  AddBasicArguments(parser_download)
+  AddBucketArguments(parser_download)
 
   # SDK Update arguments
   parser_sdk = subparsers.add_parser(
       'sdk',
-      help='update the local sdk using the Android SDK Manager',
+      help='get the latest Google Play services SDK using Android SDK Manager',
       description=UpdateSdk.__doc__,
       formatter_class=utils.DefaultsRawHelpFormatter)
-  parser_sdk.add_argument('--sdk-root',
-                          help=('base path to the Android SDK tools to use to '
-                                'update the library'),
-                          default=constants.ANDROID_SDK_ROOT)
-  parser_sdk.add_argument('-v', '--verbose',
-                          action='store_true',
-                          help='print debug information')
   parser_sdk.set_defaults(func=UpdateSdk)
+  AddBasicArguments(parser_sdk)
 
   # Upload arguments
   parser_upload = subparsers.add_parser(
@@ -91,70 +81,90 @@ def Main():
       help='upload the library to the cloud storage',
       description=Upload.__doc__,
       formatter_class=utils.DefaultsRawHelpFormatter)
-  parser_upload.add_argument('-f', '--force',
-                             action='store_true',
-                             help=('run even if the checked in version is '
-                                   'already up to date'))
-  parser_upload.add_argument('--sdk-root',
-                             help=('base path to the Android SDK tools to use '
-                                   'to update the library'),
-                             default=constants.ANDROID_SDK_ROOT)
+
   parser_upload.add_argument('--skip-git',
                              action='store_true',
                              help="don't commit the changes at the end")
   parser_upload.set_defaults(func=Upload)
-  AddCommonArguments(parser_upload)
+  AddBasicArguments(parser_upload)
+  AddBucketArguments(parser_upload)
 
-  args = parser.parse_args()
+  args = parser.parse_args(raw_args)
   if args.verbose:
     logging.basicConfig(level=logging.DEBUG)
-  logging_utils.ColorStreamHandler.MakeDefault()
+  logging_utils.ColorStreamHandler.MakeDefault(not _IsBotEnvironment())
   return args.func(args)
 
 
-def AddCommonArguments(parser):
+def AddBasicArguments(parser):
   '''
   Defines the common arguments on subparser rather than the main one. This
   allows to put arguments after the command: `foo.py upload --debug --force`
   instead of `foo.py --debug upload --force`
   '''
 
+  parser.add_argument('--sdk-root',
+                      help='base path to the Android SDK tools root',
+                      default=constants.ANDROID_SDK_ROOT)
+
+  parser.add_argument('-v', '--verbose',
+                      action='store_true',
+                      help='print debug information')
+
+
+def AddBucketArguments(parser):
   parser.add_argument('--bucket',
                       help='name of the bucket where the files are stored',
                       default=GMS_CLOUD_STORAGE)
+
   parser.add_argument('--config',
                       help='JSON Configuration file',
                       default=CONFIG_DEFAULT_PATH)
+
   parser.add_argument('--dry-run',
                       action='store_true',
                       help=('run the script in dry run mode. Files will be '
-                            'copied to a local directory instead of the cloud '
-                            'storage. The bucket name will be as path to that '
-                            'directory relative to the repository root.'))
-  parser.add_argument('-v', '--verbose',
+                            'copied to a local directory instead of the '
+                            'cloud storage. The bucket name will be as path '
+                            'to that directory relative to the repository '
+                            'root.'))
+
+  parser.add_argument('-f', '--force',
                       action='store_true',
-                      help='print debug information')
+                      help='run even if the library is already up to date')
 
 
 def Download(args):
   '''
-  Downloads the Google Play services client library from a Google Cloud Storage
-  bucket and installs it to
+  Downloads the Google Play services library from a Google Cloud Storage bucket
+  and installs it to
   //third_party/android_tools/sdk/extras/google/google_play_services.
 
   A license check will be made, and the user might have to accept the license
   if that has not been done before.
   '''
 
-  paths = _InitPaths(constants.ANDROID_SDK_ROOT)
+  if not os.path.isdir(args.sdk_root):
+    logging.debug('Did not find the Android SDK root directory at "%s".',
+                  args.sdk_root)
+    if not args.force:
+      logging.info('Skipping, not on an android checkout.')
+      return 0
+
+  paths = PlayServicesPaths(args.sdk_root)
 
-  new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY, LIBRARY_FILE_NAME + '.sha1')
-  old_lib_zip_sha1 = os.path.join(paths.package, LIBRARY_FILE_NAME + '.sha1')
+  if os.path.isdir(paths.package) and not os.access(paths.package, os.W_OK):
+    logging.error('Failed updating the Google Play Services library. '
+                  'The location is not writable. Please remove the '
+                  'directory (%s) and try again.', paths.package)
+    return -2
 
-  logging.debug('Comparing library hashes: %s and %s', new_lib_zip_sha1,
-                old_lib_zip_sha1)
-  if utils.FileEquals(new_lib_zip_sha1, old_lib_zip_sha1) and not args.force:
-    logging.debug('The Google Play services library is up to date.')
+  new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY, ZIP_FILE_NAME + '.sha1')
+
+  logging.debug('Comparing zip hashes: %s and %s', new_lib_zip_sha1,
+                paths.lib_zip_sha1)
+  if utils.FileEquals(new_lib_zip_sha1, paths.lib_zip_sha1) and not args.force:
+    logging.info('Skipping, the Google Play services library is up to date.')
     return 0
 
   config = utils.ConfigParser(args.config)
@@ -164,43 +174,53 @@ def Download(args):
 
   tmp_root = tempfile.mkdtemp()
   try:
-    if not os.environ.get('CHROME_HEADLESS'):
-      if not os.path.isdir(paths.package):
-        os.makedirs(paths.package)
-
-      # download license file from bucket/{version_number}/license.sha1
-      new_license = os.path.join(tmp_root, LICENSE_FILE_NAME)
-      old_license = os.path.join(paths.package, LICENSE_FILE_NAME)
-
-      license_sha1 = os.path.join(SHA1_DIRECTORY, LICENSE_FILE_NAME + '.sha1')
-      _DownloadFromBucket(bucket_path, license_sha1, new_license,
-                          args.verbose, args.dry_run)
-      if not _CheckLicenseAgreement(new_license, old_license):
+    # setup the destination directory
+    if not os.path.isdir(paths.package):
+      os.makedirs(paths.package)
+
+    # download license file from bucket/{version_number}/license.sha1
+    new_license = os.path.join(tmp_root, LICENSE_FILE_NAME)
+
+    license_sha1 = os.path.join(SHA1_DIRECTORY, LICENSE_FILE_NAME + '.sha1')
+    _DownloadFromBucket(bucket_path, license_sha1, new_license,
+                        args.verbose, args.dry_run)
+
+    if (not _IsBotEnvironment() and
+        not _CheckLicenseAgreement(new_license, paths.license,
+                                   config.version_number)):
         logging.warning('Your version of the Google Play services library is '
                         'not up to date. You might run into issues building '
                         'or running the app. Please run `%s download` to '
                         'retry downloading it.', __file__)
         return 0
 
-    new_lib_zip = os.path.join(tmp_root, LIBRARY_FILE_NAME)
+    new_lib_zip = os.path.join(tmp_root, ZIP_FILE_NAME)
     _DownloadFromBucket(bucket_path, new_lib_zip_sha1, new_lib_zip,
                         args.verbose, args.dry_run)
 
-    # We remove only the library itself. Users having a SDK manager installed
-    # library before will keep the documentation and samples from it.
-    shutil.rmtree(paths.lib, ignore_errors=True)
-    os.makedirs(paths.lib)
+    try:
+      # We remove the current version of the Google Play services SDK.
+      if os.path.exists(paths.package):
+        shutil.rmtree(paths.package)
+      os.makedirs(paths.package)
+
+      logging.debug('Extracting the library to %s', paths.lib)
+      with zipfile.ZipFile(new_lib_zip, "r") as new_lib_zip_file:
+        new_lib_zip_file.extractall(paths.lib)
 
-    logging.debug('Extracting the library to %s', paths.lib)
-    with zipfile.ZipFile(new_lib_zip, "r") as new_lib_zip_file:
-      new_lib_zip_file.extractall(paths.lib)
+      logging.debug('Copying %s to %s', new_license, paths.license)
+      shutil.copy(new_license, paths.license)
 
-    logging.debug('Copying %s to %s', new_license, old_license)
-    shutil.copy(new_license, old_license)
+      logging.debug('Copying %s to %s', new_lib_zip_sha1, paths.lib_zip_sha1)
+      shutil.copy(new_lib_zip_sha1, paths.lib_zip_sha1)
 
-    logging.debug('Copying %s to %s', new_lib_zip_sha1, old_lib_zip_sha1)
-    shutil.copy(new_lib_zip_sha1, old_lib_zip_sha1)
+      logging.info('Update complete.')
 
+    except Exception as e:  # pylint: disable=broad-except
+      logging.error('Failed updating the Google Play Services library. '
+                    'An error occurred while installing the new version in '
+                    'the SDK directory: %s ', e)
+      return -3
   finally:
     shutil.rmtree(tmp_root)
 
@@ -209,8 +229,8 @@ def Download(args):
 
 def UpdateSdk(args):
   '''
-  Uses the Android SDK Manager to update or download the local Google Play
-  services library. Its usual installation path is
+  Uses the Android SDK Manager to download the latest Google Play services SDK
+  locally. Its usual installation path is
   //third_party/android_tools/sdk/extras/google/google_play_services
   '''
 
@@ -229,7 +249,7 @@ def UpdateSdk(args):
 
 def Upload(args):
   '''
-  Uploads the local Google Play services client library to a Google Cloud
+  Uploads the library from the local Google Play services SDK to a Google Cloud
   storage bucket.
 
   By default, a local commit will be made at the end of the operation.
@@ -240,7 +260,7 @@ def Upload(args):
   # disable breakpad to avoid spamming the logs.
   breakpad.IS_ENABLED = False
 
-  paths = _InitPaths(args.sdk_root)
+  paths = PlayServicesPaths(args.sdk_root)
 
   if not args.skip_git and utils.IsRepoDirty(constants.DIR_SOURCE_ROOT):
     logging.error('The repo is dirty. Please commit or stash your changes.')
@@ -248,24 +268,24 @@ def Upload(args):
 
   config = utils.ConfigParser(args.config)
 
-  version_xml = os.path.join(paths.lib, 'res', 'values', 'version.xml')
-  new_version_number = utils.GetVersionNumberFromLibraryResources(version_xml)
+  new_version_number = utils.GetVersionNumberFromLibraryResources(
+      paths.version_xml)
   logging.debug('comparing versions: new=%d, old=%s',
                 new_version_number, config.version_number)
   if new_version_number <= config.version_number and not args.force:
     logging.info('The checked in version of the library is already the latest '
-                 'one. No update needed. Please rerun with --force to skip '
+                 'one. No update is needed. Please rerun with --force to skip '
                  'this check.')
     return 0
 
   tmp_root = tempfile.mkdtemp()
   try:
-    new_lib_zip = os.path.join(tmp_root, LIBRARY_FILE_NAME)
+    new_lib_zip = os.path.join(tmp_root, ZIP_FILE_NAME)
     new_license = os.path.join(tmp_root, LICENSE_FILE_NAME)
 
     # need to strip '.zip' from the file name here
     shutil.make_archive(new_lib_zip[:-4], 'zip', paths.lib)
-    _ExtractLicenseFile(new_license, paths.package)
+    _ExtractLicenseFile(new_license, paths.source_prop)
 
     bucket_path = _VerifyBucketPathFormat(args.bucket, new_version_number,
                                           args.dry_run)
@@ -274,7 +294,7 @@ def Upload(args):
     _UploadToBucket(bucket_path, files_to_upload, args.dry_run)
 
     new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY,
-                                    LIBRARY_FILE_NAME + '.sha1')
+                                    ZIP_FILE_NAME + '.sha1')
     new_license_sha1 = os.path.join(SHA1_DIRECTORY,
                                     LICENSE_FILE_NAME + '.sha1')
     shutil.copy(new_lib_zip + '.sha1', new_lib_zip_sha1)
@@ -294,35 +314,6 @@ def Upload(args):
   return 0
 
 
-def _InitPaths(sdk_root):
-  '''
-  Initializes the different paths to be used in the update process.
-  '''
-
-  PlayServicesPaths = collections.namedtuple('PlayServicesPaths', [
-      # Android SDK root path
-      'sdk_root',
-
-      # Path to the Google Play services package in the SDK manager sense,
-      # where it installs the source.properties file
-      'package',
-
-      # Path to the Google Play services library itself (jar and res)
-      'lib',
-  ])
-
-  sdk_play_services_package_dir = os.path.join('extras', 'google',
-                                               'google_play_services')
-  sdk_play_services_lib_dir = os.path.join(sdk_play_services_package_dir,
-                                           'libproject',
-                                           'google-play-services_lib')
-
-  return PlayServicesPaths(
-      sdk_root=sdk_root,
-      package=os.path.join(sdk_root, sdk_play_services_package_dir),
-      lib=os.path.join(sdk_root, sdk_play_services_lib_dir))
-
-
 def _DownloadFromBucket(bucket_path, sha1_file, destination, verbose,
                         is_dry_run):
   '''Downloads the file designated by the provided sha1 from a cloud bucket.'''
@@ -367,8 +358,7 @@ def _InitGsutil(is_dry_run):
         download_from_google_storage.GSUTIL_DEFAULT_PATH)
 
 
-def _ExtractLicenseFile(license_path, play_services_package_dir):
-  prop_file_path = os.path.join(play_services_package_dir, 'source.properties')
+def _ExtractLicenseFile(license_path, prop_file_path):
   with open(prop_file_path, 'r') as prop_file:
     prop_file_content = prop_file.read()
 
@@ -381,7 +371,8 @@ def _ExtractLicenseFile(license_path, play_services_package_dir):
     license_file.write(match.group('text'))
 
 
-def _CheckLicenseAgreement(expected_license_path, actual_license_path):
+def _CheckLicenseAgreement(expected_license_path, actual_license_path,
+                           version_number):
   '''
   Checks that the new license is the one already accepted by the user. If it
   isn't, it prompts the user to accept it. Returns whether the expected license
@@ -392,13 +383,16 @@ def _CheckLicenseAgreement(expected_license_path, actual_license_path):
     return True
 
   with open(expected_license_path) as license_file:
+    # Uses plain print rather than logging to make sure this is not formatted
+    # by the logger.
+    print ('Updating the Google Play services SDK to '
+           'version %d.' % version_number)
+
     # The output is buffered when running as part of gclient hooks. We split
     # the text here and flush is explicitly to avoid having part of it dropped
     # out.
     # Note: text contains *escaped* new lines, so we split by '\\n', not '\n'.
     for license_part in license_file.read().split('\\n'):
-      # Uses plain print rather than logging to make sure this is not formatted
-      # by the logger.
       print license_part
       sys.stdout.flush()
 
@@ -409,6 +403,10 @@ def _CheckLicenseAgreement(expected_license_path, actual_license_path):
   return raw_input('> ') in ('Y', 'y')
 
 
+def _IsBotEnvironment():
+  return bool(os.environ.get('CHROME_HEADLESS'))
+
+
 def _VerifyBucketPathFormat(bucket_name, version_number, is_dry_run):
   '''
   Formats and checks the download/upload path depending on whether we are
@@ -432,6 +430,64 @@ def _VerifyBucketPathFormat(bucket_name, version_number, is_dry_run):
   return bucket_path
 
 
+class PlayServicesPaths(object):
+  '''
+  Describes the different paths to be used in the update process.
+
+         Filesystem hierarchy                        | Exposed property / notes
+  ---------------------------------------------------|-------------------------
+  [sdk_root]                                         | sdk_root / (1)
+   +- extras                                         |
+      +- google                                      |
+         +- google_play_services                     | package / (2)
+            +- source.properties                     | source_prop / (3)
+            +- LICENSE                               | license / (4)
+            +- google_play_services_library.zip.sha1 | lib_zip_sha1 / (5)
+            +- libproject                            |
+               +- google-play-services_lib           | lib / (6)
+                  +- res                             |
+                     +- values                       |
+                        +- version.xml               | version_xml (7)
+
+  Notes:
+
+   1. sdk_root: Path provided as a parameter to the script (--sdk_root)
+   2. package: This directory contains the Google Play services SDK itself.
+      When downloaded via the Android SDK manager, it will contain,
+      documentation, samples and other files in addition to the library. When
+      the update script downloads the library from our cloud storage, it is
+      cleared.
+   3. source_prop: File created by the Android SDK manager that contains
+      the package information, such as the version info and the license.
+   4. license: File created by the update script. Contains the license accepted
+      by the user.
+   5. lib_zip_sha1: sha1 of the library zip that has been installed by the
+      update script. It is compared with the one required by the config file to
+      check if an update is necessary.
+   6. lib: Contains the library itself: jar and resources. This is what is
+      downloaded from the cloud storage.
+   7. version_xml: File that contains the exact Google Play services library
+      version, the one that we track. The version looks like 811500, is used in
+      the code and the on-device APK, as opposed to the SDK package version
+      which looks like 27.0.0 and is used only by the Android SDK manager.
+
+  '''
+
+  def __init__(self, sdk_root):
+    relative_package = os.path.join('extras', 'google', 'google_play_services')
+    relative_lib = os.path.join(relative_package, 'libproject',
+                                'google-play-services_lib')
+    self.sdk_root = sdk_root
+
+    self.package = os.path.join(sdk_root, relative_package)
+    self.lib_zip_sha1 = os.path.join(self.package, ZIP_FILE_NAME + '.sha1')
+    self.license = os.path.join(self.package, LICENSE_FILE_NAME)
+    self.source_prop = os.path.join(self.package, 'source.properties')
+
+    self.lib = os.path.join(sdk_root, relative_lib)
+    self.version_xml = os.path.join(self.lib, 'res', 'values', 'version.xml')
+
+
 class DummyGsutil(download_from_google_storage.Gsutil):
   '''
   Class that replaces Gsutil to use a local directory instead of an online
@@ -454,4 +510,4 @@ class DummyGsutil(download_from_google_storage.Gsutil):
 
 
 if __name__ == '__main__':
-  sys.exit(Main())
+  sys.exit(main(sys.argv[1:]))