Extension profile generator + benchmark for startup with profile.

tools/perf/profile_creators/update_remote_extensions.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import csv
+import optparse
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+import zipfile
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+    'telemetry'))
+
+from catapult_base import cloud_storage
+from telemetry.core import exceptions
+
+# Remote target upload directory in cloud storage for extensions.
+REMOTE_DIR = 'extension_set'
+
+# Target zip file.
+ZIP_NAME = 'extensions.zip'
+
+def _DownloadCrxFromCws(ext_id, dst):
+  """Downloads CRX specified from Chrome Web Store.
+
+  Retrieves CRX (Chrome extension file) specified by ext_id from Chrome Web
+  Store, into directory specified by dst.

[nednguyen, 2015/07/17 21:48:07]

Does this CRX file have a fixed version? 

[sydli, 2015/07/17 22:29:27]

No, should it be? update_remote_extensions will not be run very often, so any
generated profile will have consistent extension versions.

[nednguyen, 2015/07/17 22:49:08]

I see. In case someone runs "update_remote_extensions", would these binary be
update silently? Your design should make sure that any changing parts in the
system is done through a CL. That would allow easy rollback when anything goes
wrong & the bisect bot can blame the CL if it causes regression.

My suggestion is that you should also hash the extension binary and make it part
of the keys in _DownloadExtensionFromCloud(key)

[sydli, 2015/07/20 21:33:41]

Done; not sure if this is exactly what you meant, but now stores CRX hash +
version in CSV. If anything goes wrong, can inspect CL for individual extension
version/hash changes in CSV.

+
+  Args:
+      ext_id: id of extension to retrieve.
+      dst: directory to download CRX into
+
+  Returns:
+      Returns local path to downloaded CRX.
+      If download fails, return None.
+  """
+  dst_path = os.path.join(dst, '%s.crx' % ext_id)
+  cws_url = ('https://clients2.google.com/service/update2/crx?response='
+            'redirect&prodversion=38.0&x=id%%3D%s%%26installsource%%3D'
+            'ondemand%%26uc' % ext_id)
+  response = urllib2.urlopen(cws_url)
+  if response.getcode() is not 200:
+    return None
+  with open(dst_path, 'w') as f:
+    f.write(response.read())
+  return dst_path
+
+def _UpdateExtensionsInCloud(local_extensions_dir, extensions_csv, remote_dir):
+  """Updates set of extensions in Cloud Storage from a CSV of extension ids.
+
+  From well-formatted CSV file containing some set of extensions
+  (extensions_csv), download them, compress into archive, and update
+  the remote extension archive under REMOTE_DIR in CHROME-PARTNER-TELEMETRY
+  bucket. This script expects 2nd column of CSV file to contain extension ids.
+
+  Args:
+      local_extensions_dir: directory to download CRX files into.
+      extension_csv: CSV to pull extension_ids from.
+      remote_dir: remote directory to put extension archive in cloud storage.
+
+  Raises:
+      Exception if a CRX download fails.
+  """
+  # Download CRX to temp files and compress into archive
+  zip_path = os.path.join(local_extensions_dir, ZIP_NAME)
+  extension_zip = zipfile.ZipFile(zip_path, 'w')
+  with open(extensions_csv, 'rb') as csv_file:
+    reader = csv.reader(csv_file)
+    reader.next() # skip header line
+    for row in reader:
+      extension_id = row[1]
+      print 'Fetching extension %s...' % extension_id
+      crx_path = _DownloadCrxFromCws(extension_id, local_extensions_dir)
+      if crx_path is None:
+          raise exceptions.Error('\tCould not fetch %s.\n\n'
+                                 'If this extension dl consistently fails, '
+                                 'remove this entry from %s.'
+                                 % (extension_id, extensions_csv))
+      extension_zip.write(crx_path, arcname='%s.crx' % extension_id)
+  extension_zip.close()
+  print 'Uploading extensions to cloud...'
+  remote_zip_path = os.path.join(remote_dir, ZIP_NAME)
+  cloud_storage.Insert(cloud_storage.PARTNER_BUCKET, remote_zip_path, zip_path)
+
+def _GetCsvFromArgs():
+  """Parse options to retrieve name of CSV file."""
+  parser = optparse.OptionParser()
+  parser.add_option('-e', '--extension-csv', dest='extension_csv',
+                   help='CSV of extensions to load.')
+  (options, _) = parser.parse_args()
+  if not options.extension_csv:
+    parser.error('Must specify --extension-csv option.')
+  return options.extension_csv
+
+def main():
+  extension_csv = _GetCsvFromArgs()
+  local_extensions_dir = tempfile.mkdtemp()
+  try:
+    _UpdateExtensionsInCloud(local_extensions_dir,
+                             extension_csv, REMOTE_DIR)
+  finally:
+    shutil.rmtree(local_extensions_dir)
+
+if __name__ == '__main__':
+  main()
+