webpages_playback: use gsutil instead of API, no shell_utils

tools/skp/webpages_playback.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Archives or replays webpages and creates SKPs in a Google Storage location.
 
 To archive webpages and store SKP files (archives should be rarely updated):
 
 cd skia
@@ skipping 45 matching lines @@
 import optparse
 import os
 import posixpath
 import shutil
 import subprocess
 import sys
 import tempfile
 import time
 import traceback
 
-SKIA_DIR = os.path.abspath(os.path.join(
-    os.path.realpath(os.path.dirname(__file__)),
-    os.pardir, os.pardir))
-sys.path.insert(0, SKIA_DIR)
-
-from common.py.utils import gs_utils
-from common.py.utils import shell_utils
 
 ROOT_PLAYBACK_DIR_NAME = 'playback'
 SKPICTURES_DIR_NAME = 'skps'
 
+GS_PREFIX = 'gs://'
+
 PARTNERS_GS_BUCKET = 'gs://chrome-partner-telemetry'
 
 # Local archive and SKP directories.
 LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join(
     os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data')
 TMP_SKP_DIR = tempfile.mkdtemp()
 
 # Location of the credentials.json file and the string that represents missing
 # passwords.
 CREDENTIALS_FILE_PATH = os.path.join(
@@ skipping 17 matching lines @@
 # How many times the record_wpr binary should be retried.
 RETRY_RECORD_WPR_COUNT = 5
 # How many times the run_benchmark binary should be retried.
 RETRY_RUN_MEASUREMENT_COUNT = 5
 
 # Location of the credentials.json file in Google Storage.
 CREDENTIALS_GS_PATH = '/playback/credentials/credentials.json'
 
 X11_DISPLAY = os.getenv('DISPLAY', ':0')
 
-GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE
-GS_FINE_GRAINED_ACL_LIST = [
-  (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com',
-   gs_utils.GSUtils.Permission.READ),
-]
-
 # Path to Chromium's page sets.
 CHROMIUM_PAGE_SETS_PATH = os.path.join('tools', 'perf', 'page_sets')
 
 # Dictionary of supported Chromium page sets to their file prefixes.
 CHROMIUM_PAGE_SETS_TO_PREFIX = {
     'key_mobile_sites_smooth.py': 'keymobi',
     'top_25_smooth.py': 'top25desk',
 }
 
 
@@ skipping 19 matching lines @@
                                                CHROMIUM_PAGE_SETS_PATH)
     self._all_page_sets_specified = parse_options.page_sets == 'all'
     self._page_sets = self._ParsePageSets(parse_options.page_sets)
 
     self._record = parse_options.record
     self._skia_tools = parse_options.skia_tools
     self._non_interactive = parse_options.non_interactive
     self._upload = parse_options.upload
     self._skp_prefix = parse_options.skp_prefix
     data_store_location = parse_options.data_store
-    if data_store_location.startswith(gs_utils.GS_PREFIX):
+    if data_store_location.startswith(GS_PREFIX):
       self.gs = GoogleStorageDataStore(data_store_location)
     else:
       self.gs = LocalFileSystemDataStore(data_store_location)
     self._upload_to_partner_bucket = parse_options.upload_to_partner_bucket
     self._alternate_upload_dir = parse_options.alternate_upload_dir
     self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path,
                                                 'tools', 'perf')
     self._catapult_dir = os.path.join(parse_options.chrome_src_path,
                                       'third_party', 'catapult')
 
@@ skipping 84 matching lines @@
           'DISPLAY=%s' % X11_DISPLAY,
           os.path.join(self._telemetry_binaries_dir, 'record_wpr'),
           '--extra-browser-args="%s"' % self._browser_args,
           '--browser=exact',
           '--browser-executable=%s' % self._browser_executable,
           '%s_page_set' % page_set_basename,
           '--page-set-base-dir=%s' % page_set_dir
         )
         for _ in range(RETRY_RECORD_WPR_COUNT):
           try:
-            shell_utils.run(' '.join(record_wpr_cmd), shell=True)
+            subprocess.check_call(' '.join(record_wpr_cmd), shell=True)
 
             # Move over the created archive into the local webpages archive
             # directory.
             shutil.move(
               os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
               self._local_record_webpages_archive_dir)
             shutil.move(
               os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
                            page_set_json_name),
               self._local_record_webpages_archive_dir)
@@ skipping 23 matching lines @@
           SKP_BENCHMARK,
           '--page-set-name=%s' % page_set_basename,
           '--page-set-base-dir=%s' % page_set_dir,
           '--skp-outdir=%s' % TMP_SKP_DIR,
           '--also-run-disabled-tests'
       )
 
       for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
         try:
           print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
-          shell_utils.run(' '.join(run_benchmark_cmd), shell=True)
-        except shell_utils.CommandFailedException:
+          subprocess.check_call(' '.join(run_benchmark_cmd), shell=True)
+        except subprocess.CalledProcessError:
           # skpicture_printer sometimes fails with AssertionError but the
           # captured SKP is still valid. This is a known issue.
           pass
 
         # Rename generated SKP files into more descriptive names.
         try:
           self._RenameSkpFiles(page_set)
           # Break out of the retry loop since there were no errors.
           break
         except Exception:
@@ skipping 14 matching lines @@
           os.path.join(self._skia_tools, 'render_pictures'),
           '-r', self._local_skp_dir
       ]
       render_pdfs_cmd = [
           os.path.join(self._skia_tools, 'render_pdfs'),
           '-r', self._local_skp_dir
       ]
 
       for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
         print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
-        proc = subprocess.Popen(tools_cmd)
-        (code, _) = shell_utils.log_process_after_completion(proc, echo=False)
-        if code != 0:
-          raise Exception('%s failed!' % ' '.join(tools_cmd))
+        subprocess.check_call(tools_cmd)
 
       if not self._non_interactive:
         print '\n\n=======Running debugger======='
         os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
                              self._local_skp_dir))
 
     print '\n\n'
 
     if self._upload:
       print '\n\n=======Uploading to %s=======\n\n' % self.gs.target_type()
       # Copy the directory structure in the root directory into Google Storage.
       dest_dir_name = ROOT_PLAYBACK_DIR_NAME
       if self._alternate_upload_dir:
         dest_dir_name = self._alternate_upload_dir
 
       self.gs.upload_dir_contents(
-          self._local_skp_dir, dest_dir=dest_dir_name,
-          upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED,
-          predefined_acl=GS_PREDEFINED_ACL,
-          fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST)
+          self._local_skp_dir, dest_dir=dest_dir_name)
 
       print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
           posixpath.join(self.gs.target_name(), dest_dir_name,
                          SKPICTURES_DIR_NAME))
 
     else:
       print '\n\n=======Not Uploading to %s=======\n\n' % self.gs.target_type()
       print 'Generated resources are available in %s\n\n' % (
           self._local_skp_dir)
 
     if self._upload_to_partner_bucket:
       print '\n\n=======Uploading to Partner bucket %s =======\n\n' % (
           PARTNERS_GS_BUCKET)
       partner_gs = GoogleStorageDataStore(PARTNERS_GS_BUCKET)
       partner_gs.delete_path(SKPICTURES_DIR_NAME)
       print 'Uploading %s to %s' % (self._local_skp_dir, SKPICTURES_DIR_NAME)
-      partner_gs.upload_dir_contents(
-          self._local_skp_dir,
-          dest_dir=SKPICTURES_DIR_NAME,
-          upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED)
+      partner_gs.upload_dir_contents(self._local_skp_dir, SKPICTURES_DIR_NAME)
       print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
           posixpath.join(partner_gs.target_name(), SKPICTURES_DIR_NAME))
 
     return 0
 
   def _GetSkiaSkpFileName(self, page_set):
     """Returns the SKP file name for Skia page sets."""
     # /path/to/skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop.py
     ps_filename = os.path.basename(page_set)
     # skia_yahooanswers_desktop.py -> skia_yahooanswers_desktop
@@ skipping 71 matching lines @@
       raise Exception('%s and %s do not exist in %s!' % (gs.target_type(),
         wpr_source, page_set_source))
 
 class DataStore:
   """An abstract base class for uploading recordings to a data storage.
   The interface emulates the google storage api."""
   def target_name(self):
     raise NotImplementedError()
   def target_type(self):
     raise NotImplementedError()
-  def does_storage_object_exist(self, *args):
+  def does_storage_object_exist(self, name):
     raise NotImplementedError()
-  def download_file(self, *args):
+  def download_file(self, name, local_path):
     raise NotImplementedError()
-  def upload_dir_contents(self, source_dir, **kwargs):
+  def upload_dir_contents(self, source_dir, dest_dir):
     raise NotImplementedError()
 
+
 class GoogleStorageDataStore(DataStore):
   def __init__(self, data_store_url):
-    self._data_store_url = data_store_url
-    self._bucket = remove_prefix(self._data_store_url.lstrip(),
-                                 gs_utils.GS_PREFIX)
-    self.gs = gs_utils.GSUtils()
+    self._url = data_store_url
+
   def target_name(self):
-    return self._data_store_url
+    return self._url
+
   def target_type(self):
     return 'Google Storage'
-  def does_storage_object_exist(self, *args):
-    return self.gs.does_storage_object_exist(self._bucket, *args)
+
+  def does_storage_object_exist(self, name):
+    try:
+      output = subprocess.check_output([
+          'gsutil', 'ls', '/'.join((self._url, name))])
+    except subprocess.CalledProcessError:
+      return False
+    if len(output.splitlines) != 1:
+      return False
+    return True
+
   def delete_path(self, path):
-    _, files = self.gs.list_bucket_contents(self._bucket, subdir=path)
-    for f in files:
-      self.gs.delete_file(self._bucket, posixpath.join(path, f))
-  def download_file(self, *args):
-    self.gs.download_file(self._bucket, *args)
-  def upload_dir_contents(self, source_dir, **kwargs):
-    self.gs.upload_dir_contents(source_dir, self._bucket, **kwargs)
+    subprocess.check_call(['gsutil', 'rm', '-r', path])
+
+  def download_file(self, name, local_path):
+    subprocess.check_call([
+        'gsutil', 'cp', '/'.join((self._url, name)), local_path])
+
+  def upload_dir_contents(self, source_dir, dest_dir):
+    subprocess.check_call([
+        'gsutil', 'cp', '-r', source_dir, '/'.join(self._url, dest_dir)])
+
 
 class LocalFileSystemDataStore(DataStore):
   def __init__(self, data_store_location):
     self._base_dir = data_store_location
   def target_name(self):
     return self._base_dir
   def target_type(self):
     return self._base_dir
-  def does_storage_object_exist(self, name, *args):
+  def does_storage_object_exist(self, name):
     return os.path.isfile(os.path.join(self._base_dir, name))
   def delete_path(self, path):
     shutil.rmtree(path)
-  def download_file(self, name, local_path, *args):
+  def download_file(self, name, local_path):
     shutil.copyfile(os.path.join(self._base_dir, name), local_path)
-  def upload_dir_contents(self, source_dir, dest_dir, **kwargs):
+  def upload_dir_contents(self, source_dir, dest_dir):
     def copytree(source_dir, dest_dir):
       if not os.path.exists(dest_dir):
         os.makedirs(dest_dir)
       for item in os.listdir(source_dir):
         source = os.path.join(source_dir, item)
         dest = os.path.join(dest_dir, item)
         if os.path.isdir(source):
           copytree(source, dest)
         else:
           shutil.copy2(source, dest)
@@ skipping 62 matching lines @@
            '--skia_tools is specified then the debugger is not run.',
       default=False)
   option_parser.add_option(
       '', '--skp_prefix',
       help='Prefix to add to the names of generated SKPs.',
       default=None)
   options, unused_args = option_parser.parse_args()
 
   playback = SkPicturePlayback(options)
   sys.exit(playback.Run())