build/android/play_services/update.py - Issue 1418833005: [gms updater] Fixes and tests to prepare activation

Unified Diff: build/android/play_services/update.py

Issue 1418833005: [gms updater] Fixes and tests to prepare activation (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master

Patch Set: More doc, clean up naming Created 5 years, 1 month ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

Index: build/android/play_services/update.py

diff --git a/build/android/play_services/update.py b/build/android/play_services/update.py

index b2d57ad359f49c7c271246d4f0b629b332b551d0..db237daf59b0677c3b21d0b82c44bb5d1682d071 100755

--- a/build/android/play_services/update.py

+++ b/build/android/play_services/update.py

@@ -4,12 +4,11 @@

# found in the LICENSE file.

'''

-Script to help uploading and downloading the Google Play services client

-library to and from a Google Cloud storage.

+Script to help uploading and downloading the Google Play services SDK to and

jbudorick 2015/11/05 16:03:02 nit: Why the doc change from "library" to "SDK"? L

dgn 2015/11/05 16:15:05 The developer website says "Google Play services S

dgn 2015/11/05 16:30:21 Done.

+from a Google Cloud storage.

'''

import argparse

-import collections

import logging

import os

import re

@@ -37,21 +36,21 @@ SHA1_DIRECTORY = os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'android',

'play_services')

# Default bucket used for storing the files.

-GMS_CLOUD_STORAGE = 'chrome-sdk-extras'

+GMS_CLOUD_STORAGE = 'chromium-android-tools/play-services'

# Path to the default configuration file. It exposes the currently installed

-# version of the library in a human readable way.

+# version of the SDK in a human readable way.

CONFIG_DEFAULT_PATH = os.path.join(constants.DIR_SOURCE_ROOT, 'build',

'android', 'play_services', 'config.json')

LICENSE_FILE_NAME = 'LICENSE'

-LIBRARY_FILE_NAME = 'google_play_services_library.zip'

+ZIP_FILE_NAME = 'google_play_services_library.zip'

GMS_PACKAGE_ID = 'extra-google-google_play_services' # used by sdk manager

LICENSE_PATTERN = re.compile(r'^Pkg\.License=(?P<text>.*)$', re.MULTILINE)

-def Main():

+def main(raw_args):

parser = argparse.ArgumentParser(

description=__doc__ + 'Please see the subcommand help for more details.',

formatter_class=utils.DefaultsRawHelpFormatter)

@@ -60,101 +59,112 @@ def Main():

# Download arguments

parser_download = subparsers.add_parser(

'download',

- help='download the library from the cloud storage',

+ help='download the SDK from the cloud storage',

description=Download.__doc__,

formatter_class=utils.DefaultsRawHelpFormatter)

- parser_download.add_argument('-f', '--force',

- action='store_true',

- help=('run even if the local version is '

- 'already up to date'))

parser_download.set_defaults(func=Download)

- AddCommonArguments(parser_download)

+ AddBasicArguments(parser_download)

+ AddBucketArguments(parser_download)

# SDK Update arguments

parser_sdk = subparsers.add_parser(

'sdk',

- help='update the local sdk using the Android SDK Manager',

+ help='update the SDK using the Android SDK Manager',

description=UpdateSdk.__doc__,

formatter_class=utils.DefaultsRawHelpFormatter)

- parser_sdk.add_argument('--sdk-root',

- help=('base path to the Android SDK tools to use to '

- 'update the library'),

- default=constants.ANDROID_SDK_ROOT)

- parser_sdk.add_argument('-v', '--verbose',

- action='store_true',

- help='print debug information')

parser_sdk.set_defaults(func=UpdateSdk)

+ AddBasicArguments(parser_sdk)

# Upload arguments

parser_upload = subparsers.add_parser(

'upload',

- help='upload the library to the cloud storage',

+ help='upload the SDK to the cloud storage',

description=Upload.__doc__,

formatter_class=utils.DefaultsRawHelpFormatter)

- parser_upload.add_argument('-f', '--force',

- action='store_true',

- help=('run even if the checked in version is '

- 'already up to date'))

- parser_upload.add_argument('--sdk-root',

- help=('base path to the Android SDK tools to use '

- 'to update the library'),

- default=constants.ANDROID_SDK_ROOT)

parser_upload.add_argument('--skip-git',

action='store_true',

help="don't commit the changes at the end")

parser_upload.set_defaults(func=Upload)

- AddCommonArguments(parser_upload)

+ AddBasicArguments(parser_upload)

+ AddBucketArguments(parser_upload)

- args = parser.parse_args()

+ args = parser.parse_args(raw_args)

if args.verbose:

logging.basicConfig(level=logging.DEBUG)

- logging_utils.ColorStreamHandler.MakeDefault()

+ logging_utils.ColorStreamHandler.MakeDefault(not _IsBotEnvironment())

return args.func(args)

-def AddCommonArguments(parser):

+def AddBasicArguments(parser):

'''

Defines the common arguments on subparser rather than the main one. This

allows to put arguments after the command: `foo.py upload --debug --force`

instead of `foo.py --debug upload --force`

'''

+ parser.add_argument('--sdk-root',

+ help='base path to the Android SDK tools root',

+ default=constants.ANDROID_SDK_ROOT)

+ parser.add_argument('-v', '--verbose',

+ action='store_true',

+ help='print debug information')

+def AddBucketArguments(parser):

parser.add_argument('--bucket',

help='name of the bucket where the files are stored',

default=GMS_CLOUD_STORAGE)

parser.add_argument('--config',

help='JSON Configuration file',

default=CONFIG_DEFAULT_PATH)

parser.add_argument('--dry-run',

action='store_true',

help=('run the script in dry run mode. Files will be '

- 'copied to a local directory instead of the cloud '

- 'storage. The bucket name will be as path to that '

- 'directory relative to the repository root.'))

- parser.add_argument('-v', '--verbose',

+ 'copied to a local directory instead of the '

+ 'cloud storage. The bucket name will be as path '

+ 'to that directory relative to the repository '

+ 'root.'))

+ parser.add_argument('-f', '--force',

action='store_true',

- help='print debug information')

+ help='run even if the SDK is already up to date')

def Download(args):

'''

- Downloads the Google Play services client library from a Google Cloud Storage

- bucket and installs it to

+ Downloads the Google Play services SDK from a Google Cloud Storage bucket and

+ installs it to

//third_party/android_tools/sdk/extras/google/google_play_services.

A license check will be made, and the user might have to accept the license

if that has not been done before.

'''

- paths = _InitPaths(constants.ANDROID_SDK_ROOT)

+ if not os.path.isdir(args.sdk_root):

+ logging.debug('Did not find the Android SDK root directory at "%s".',

+ args.sdk_root)

+ if not args.force:

+ logging.info('Skipping, not on an android checkout.')

+ return 0

+ paths = PlayServicesPaths(args.sdk_root)

- new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY, LIBRARY_FILE_NAME + '.sha1')

- old_lib_zip_sha1 = os.path.join(paths.package, LIBRARY_FILE_NAME + '.sha1')

+ if os.path.isdir(paths.package) and not os.access(paths.package, os.W_OK):

+ logging.error('Failed updating the Google Play Services SDK. '

+ 'The location is not writable. Please remove the '

+ 'directory (%s) and try again.', paths.package)

+ return -2

- logging.debug('Comparing library hashes: %s and %s', new_lib_zip_sha1,

- old_lib_zip_sha1)

- if utils.FileEquals(new_lib_zip_sha1, old_lib_zip_sha1) and not args.force:

- logging.debug('The Google Play services library is up to date.')

+ new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY, ZIP_FILE_NAME + '.sha1')

+ logging.debug('Comparing zip hashes: %s and %s', new_lib_zip_sha1,

+ paths.lib_zip_sha1)

+ if utils.FileEquals(new_lib_zip_sha1, paths.lib_zip_sha1) and not args.force:

+ logging.info('Skipping, the Google Play services SDK is up to date.')

return 0

config = utils.ConfigParser(args.config)

@@ -164,43 +174,60 @@ def Download(args):

tmp_root = tempfile.mkdtemp()

try:

- if not os.environ.get('CHROME_HEADLESS'):

- if not os.path.isdir(paths.package):

- os.makedirs(paths.package)

- # download license file from bucket/{version_number}/license.sha1

- new_license = os.path.join(tmp_root, LICENSE_FILE_NAME)

- old_license = os.path.join(paths.package, LICENSE_FILE_NAME)

- license_sha1 = os.path.join(SHA1_DIRECTORY, LICENSE_FILE_NAME + '.sha1')

- _DownloadFromBucket(bucket_path, license_sha1, new_license,

- args.verbose, args.dry_run)

- if not _CheckLicenseAgreement(new_license, old_license):

- logging.warning('Your version of the Google Play services library is '

+ # setup the destination directory

+ if not os.path.isdir(paths.package):

+ os.makedirs(paths.package)

+ # download license file from bucket/{version_number}/license.sha1

+ new_license = os.path.join(tmp_root, LICENSE_FILE_NAME)

+ license_sha1 = os.path.join(SHA1_DIRECTORY, LICENSE_FILE_NAME + '.sha1')

+ _DownloadFromBucket(bucket_path, license_sha1, new_license,

+ args.verbose, args.dry_run)

+ if (not _IsBotEnvironment() and

+ not _CheckLicenseAgreement(new_license, paths.license,

+ config.version_number)):

+ logging.warning('Your version of the Google Play services SDK is '

'not up to date. You might run into issues building '

'or running the app. Please run `%s download` to '

'retry downloading it.', __file__)

return 0

- new_lib_zip = os.path.join(tmp_root, LIBRARY_FILE_NAME)

+ new_lib_zip = os.path.join(tmp_root, ZIP_FILE_NAME)

_DownloadFromBucket(bucket_path, new_lib_zip_sha1, new_lib_zip,

args.verbose, args.dry_run)

- # We remove only the library itself. Users having a SDK manager installed

- # library before will keep the documentation and samples from it.

- shutil.rmtree(paths.lib, ignore_errors=True)

- os.makedirs(paths.lib)

+ try:

+ # We remove the current version of the Google Play services SDK. Users

+ # having installed it with the Android SDK manager before will keep the

+ # documentation and samples from it.

+ if os.path.exists(paths.lib):

+ shutil.rmtree(paths.lib)

+ os.makedirs(paths.lib)

+ # We also remove source.properties so that the SDK manager doesn't fail

+ # recognizing that it didn't install the local version.

+ if os.path.isfile(paths.source_prop):

+ os.remove(paths.source_prop)

- logging.debug('Extracting the library to %s', paths.lib)

- with zipfile.ZipFile(new_lib_zip, "r") as new_lib_zip_file:

- new_lib_zip_file.extractall(paths.lib)

+ logging.debug('Extracting the SDK to %s', paths.lib)

+ with zipfile.ZipFile(new_lib_zip, "r") as new_lib_zip_file:

+ new_lib_zip_file.extractall(paths.lib)

- logging.debug('Copying %s to %s', new_license, old_license)

- shutil.copy(new_license, old_license)

+ logging.debug('Copying %s to %s', new_license, paths.license)

+ shutil.copy(new_license, paths.license)

- logging.debug('Copying %s to %s', new_lib_zip_sha1, old_lib_zip_sha1)

- shutil.copy(new_lib_zip_sha1, old_lib_zip_sha1)

+ logging.debug('Copying %s to %s', new_lib_zip_sha1, paths.lib_zip_sha1)

+ shutil.copy(new_lib_zip_sha1, paths.lib_zip_sha1)

+ logging.info('Update complete.')

+ except Exception as e: # pylint: disable=broad-except

+ logging.error('Failed updating the Google Play Services SDK. '

+ 'An error occurred while installing the new version in '

+ 'the SDK directory: %s ', e)

+ return -3

finally:

shutil.rmtree(tmp_root)

@@ -210,7 +237,7 @@ def Download(args):

def UpdateSdk(args):

'''

Uses the Android SDK Manager to update or download the local Google Play

- services library. Its usual installation path is

+ services SDK. Its usual installation path is

//third_party/android_tools/sdk/extras/google/google_play_services

'''

@@ -229,7 +256,7 @@ def UpdateSdk(args):

def Upload(args):

'''

- Uploads the local Google Play services client library to a Google Cloud

+ Uploads the library from the local Google Play services SDK to a Google Cloud

storage bucket.

By default, a local commit will be made at the end of the operation.

@@ -240,7 +267,7 @@ def Upload(args):

# disable breakpad to avoid spamming the logs.

breakpad.IS_ENABLED = False

- paths = _InitPaths(args.sdk_root)

+ paths = PlayServicesPaths(args.sdk_root)

if not args.skip_git and utils.IsRepoDirty(constants.DIR_SOURCE_ROOT):

logging.error('The repo is dirty. Please commit or stash your changes.')

@@ -248,24 +275,24 @@ def Upload(args):

config = utils.ConfigParser(args.config)

- version_xml = os.path.join(paths.lib, 'res', 'values', 'version.xml')

- new_version_number = utils.GetVersionNumberFromLibraryResources(version_xml)

+ new_version_number = utils.GetVersionNumberFromLibraryResources(

+ paths.version_xml)

logging.debug('comparing versions: new=%d, old=%s',

new_version_number, config.version_number)

if new_version_number <= config.version_number and not args.force:

- logging.info('The checked in version of the library is already the latest '

+ logging.info('The checked in version of the SDK is already the latest '

'one. No update needed. Please rerun with --force to skip '

'this check.')

return 0

tmp_root = tempfile.mkdtemp()

try:

- new_lib_zip = os.path.join(tmp_root, LIBRARY_FILE_NAME)

+ new_lib_zip = os.path.join(tmp_root, ZIP_FILE_NAME)

new_license = os.path.join(tmp_root, LICENSE_FILE_NAME)

# need to strip '.zip' from the file name here

shutil.make_archive(new_lib_zip[:-4], 'zip', paths.lib)

- _ExtractLicenseFile(new_license, paths.package)

+ _ExtractLicenseFile(new_license, paths.source_prop)

bucket_path = _VerifyBucketPathFormat(args.bucket, new_version_number,

args.dry_run)

@@ -274,7 +301,7 @@ def Upload(args):

_UploadToBucket(bucket_path, files_to_upload, args.dry_run)

new_lib_zip_sha1 = os.path.join(SHA1_DIRECTORY,

- LIBRARY_FILE_NAME + '.sha1')

+ ZIP_FILE_NAME + '.sha1')

new_license_sha1 = os.path.join(SHA1_DIRECTORY,

LICENSE_FILE_NAME + '.sha1')

shutil.copy(new_lib_zip + '.sha1', new_lib_zip_sha1)

@@ -294,35 +321,6 @@ def Upload(args):

return 0

-def _InitPaths(sdk_root):

- '''

- Initializes the different paths to be used in the update process.

- '''

- PlayServicesPaths = collections.namedtuple('PlayServicesPaths', [

- # Android SDK root path

- 'sdk_root',

- # Path to the Google Play services package in the SDK manager sense,

- # where it installs the source.properties file

- 'package',

- # Path to the Google Play services library itself (jar and res)

- 'lib',

- ])

- sdk_play_services_package_dir = os.path.join('extras', 'google',

- 'google_play_services')

- sdk_play_services_lib_dir = os.path.join(sdk_play_services_package_dir,

- 'libproject',

- 'google-play-services_lib')

- return PlayServicesPaths(

- sdk_root=sdk_root,

- package=os.path.join(sdk_root, sdk_play_services_package_dir),

- lib=os.path.join(sdk_root, sdk_play_services_lib_dir))

def _DownloadFromBucket(bucket_path, sha1_file, destination, verbose,

is_dry_run):

'''Downloads the file designated by the provided sha1 from a cloud bucket.'''

@@ -367,8 +365,7 @@ def _InitGsutil(is_dry_run):

download_from_google_storage.GSUTIL_DEFAULT_PATH)

-def _ExtractLicenseFile(license_path, play_services_package_dir):

- prop_file_path = os.path.join(play_services_package_dir, 'source.properties')

+def _ExtractLicenseFile(license_path, prop_file_path):

with open(prop_file_path, 'r') as prop_file:

prop_file_content = prop_file.read()

@@ -381,7 +378,8 @@ def _ExtractLicenseFile(license_path, play_services_package_dir):

license_file.write(match.group('text'))

-def _CheckLicenseAgreement(expected_license_path, actual_license_path):

+def _CheckLicenseAgreement(expected_license_path, actual_license_path,

+ version_number):

'''

Checks that the new license is the one already accepted by the user. If it

isn't, it prompts the user to accept it. Returns whether the expected license

@@ -392,13 +390,16 @@ def _CheckLicenseAgreement(expected_license_path, actual_license_path):

return True

with open(expected_license_path) as license_file:

+ # Uses plain print rather than logging to make sure this is not formatted

+ # by the logger.

+ print ('Updating the Google Play services SDK to '

+ 'version %d.' % version_number)

# The output is buffered when running as part of gclient hooks. We split

# the text here and flush is explicitly to avoid having part of it dropped

# out.

# Note: text contains *escaped* new lines, so we split by '\\n', not '\n'.

for license_part in license_file.read().split('\\n'):

- # Uses plain print rather than logging to make sure this is not formatted

- # by the logger.

print license_part

sys.stdout.flush()

@@ -409,6 +410,10 @@ def _CheckLicenseAgreement(expected_license_path, actual_license_path):

return raw_input('> ') in ('Y', 'y')

+def _IsBotEnvironment():

+ return bool(os.environ.get('CHROME_HEADLESS'))

def _VerifyBucketPathFormat(bucket_name, version_number, is_dry_run):

'''

Formats and checks the download/upload path depending on whether we are

@@ -432,6 +437,64 @@ def _VerifyBucketPathFormat(bucket_name, version_number, is_dry_run):

return bucket_path

+class PlayServicesPaths(object):

+ '''

+ Describes the different paths to be used in the update process.

+ Filesystem hierarchy | Exposed property / notes

+ ---------------------------------------------------|-------------------------

+ [sdk_root] | sdk_root / (1)

+ +- extras |

+ +- google |

+ +- google_play_services | package / (2)

+ +- source.properties | source_prop / (3)

+ +- LICENSE | license / (4)

+ +- google_play_services_library.zip.sha1 | lib_zip_sha1 / (5)

+ +- libproject |

+ +- google-play-services_lib | lib / (6)

+ +- res |

+ +- values |

+ +- version.xml | version_xml (7)

+ Notes:

+ 1. sdk_root: Path provided as a parameter to the script (--sdk_root)

+ 2. package: This directory contains the Google Play services SDK itself.

+ When downloaded via the Android SDK manager, it will contain,

+ documentation, samples and other files in addition to the library.

+ 3. source_prop: File created by the Android SDK manager that contains

+ the package information, such as the version info and the license. When

+ the update script downloads the SDK from our cloud storage, it is

+ removed.

+ 4. license: File created by the update script. Contains the license accepted

+ by the user.

+ 5. lib_zip_sha1: sha1 of the SDK zip that has been installed by the updated

+ script. It is compared with the one required by the config file to check

+ if an update is necessary.

+ 6. lib: Contains the library itself: jar and resources. This is what is

+ downloaded from the cloud storage.

+ 7. version_xml: File that contains the exact Google Play services library

+ version, the one that we track. The version looks like 811500, is used in

+ the code and the on-device APK, as opposed to the SDK package version

+ which looks like 27.0.0 and is used only by the Android SDK manager.

+ '''

+ def __init__(self, sdk_root):

+ relative_package = os.path.join('extras', 'google', 'google_play_services')

+ relative_lib = os.path.join(relative_package, 'libproject',

+ 'google-play-services_lib')

+ self.sdk_root = sdk_root

+ self.package = os.path.join(sdk_root, relative_package)

+ self.lib_zip_sha1 = os.path.join(self.package, ZIP_FILE_NAME + '.sha1')

+ self.license = os.path.join(self.package, LICENSE_FILE_NAME)

+ self.source_prop = os.path.join(self.package, 'source.properties')

+ self.lib = os.path.join(sdk_root, relative_lib)

+ self.version_xml = os.path.join(self.lib, 'res', 'values', 'version.xml')

class DummyGsutil(download_from_google_storage.Gsutil):

'''

Class that replaces Gsutil to use a local directory instead of an online

@@ -454,4 +517,4 @@ class DummyGsutil(download_from_google_storage.Gsutil):

if __name__ == '__main__':

- sys.exit(Main())

+ sys.exit(main(sys.argv[1:]))

« no previous file with comments | « build/android/PRESUBMIT.py ('k') | build/android/play_services/update_test.py » ('j') | no next file with comments »