Extension profile generator + benchmark for startup with profile.

tools/perf/profile_creators/update_remote_extensions.py

+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import csv
+import optparse
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+import zipfile
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+    'telemetry'))
+
+from catapult_base import cloud_storage
+
+
+# Remote directory to upload extensions to in cloud storage.
+REMOTE_DIR = "extension_set"
+# Name of zip archive to write to
+ZIP_NAME = "extensions.zip"
+
+def _DownloadCRXFromCWS(ext_id, dst):
+  """ Downloads CRX specified from Chrome Web Store.
+
+  Retrieves CRX (Chrome extension file) sepcified by ext_id from Chrome Web
+  Store, into directory specified by dst.
+
+  Args:
+      ext_id: id of extension to retrieve.
+      dst: directory to download CRX into
+
+  Returns:
+      If retreiving CRX fails, returns None. Otherwise returns local path
+      to downloaded CRX. """
+  dst_path = os.path.join(dst, "%s.crx" % ext_id)
+  cws_url = ("https://clients2.google.com/service/update2/crx?response="
+          "redirect&prodversion=38.0&x=id%%3D%s%%26installsource%%3D"
+          "ondemand%%26uc" % ext_id)
+  response = urllib2.urlopen(cws_url)
+  if response.getcode() is not 200:
+      return None

[erikchen, 2015/07/16 00:33:28]

nit: too many spaces.

[sydli, 2015/07/16 00:53:14]

Done.

+  with open(dst_path, "w") as f:
+    f.write(response.read())
+  return dst_path
+
+def _UpdateExtensionsInCloud(local_extensions_dir, extensions_csv, remote_dir):
+  """ Updates set of extensions in Cloud Storage from CSV of extension ids.
+
+  From well-formatted CSV file containing some set of extensions
+  (extensions_csv), download them, compress into archive, and update
+  the remote extension archive under REMOTE_DIR in CHROME-PARTNER-TELEMETRY
+  bucket. This script expects 2nd column of CSV file to contain extension ids.
+
+  Args:
+      local_extensions_dir: directory to download CRX files into.
+      extension_csv: CSV to pull extension_ids from.
+      remote_dir: remote directory to put extension archive in cloud storage.
+  """
+  # Download CRX to temp files and compress into archive
+  zip_path = os.path.join(local_extensions_dir, ZIP_NAME)
+  extension_zip = zipfile.ZipFile(zip_path, 'w')
+  with open(extensions_csv, 'rb') as csvfile:
+    reader = csv.reader(csvfile)
+    reader.next() # skip header line
+    for row in reader:
+      extension_id = row[1]
+      print "Fetching extension %s..." % extension_id
+      crx_path = _DownloadCRXFromCWS(extension_id, local_extensions_dir)
+      if crx_path is None:

[erikchen, 2015/07/16 00:33:28]

Under this code, it's possible that there's a good zip in the cloud, that the
newly generated zip has some failures, but the newly generated zip replaces the
clodu zip anyways. That is not good.

[sydli, 2015/07/16 00:53:14]

I'll raise an error instead. In the case that one of the extensions can no
longer be downloaded from CWS and the dl fails consistently (actually-- are
there cases where this happens?), the CSV must be manually updated. Will put
this in error message.

+        print "\tCould not fetch %s" % extension_id
+        continue
+      extension_zip.write(crx_path, arcname="%s.crx" % extension_id)
+  extension_zip.close()
+  print "Uploading extensions to cloud..."
+  remote_zip_path = os.path.join(remote_dir, ZIP_NAME)
+  cloud_storage.Insert(cloud_storage.PARTNER_BUCKET, remote_zip_path, zip_path)
+
+def _GetCSVFromArgs():
+  """ Parse options to retrieve name of CSV file. """
+  parser = optparse.OptionParser()
+  parser.add_option("-e", "--extension-csv", dest="extension_csv",
+          help="CSV of extensions to load.")
+  (options, _) = parser.parse_args()
+  if not options.extension_csv:
+    parser.error("Must specify --extension-csv option.")
+  return options.extension_csv
+
+def main():
+  extension_csv = _GetCSVFromArgs()
+  local_extensions_dir = tempfile.mkdtemp()
+  try:
+    _UpdateExtensionsInCloud(local_extensions_dir,
+            extension_csv, REMOTE_DIR)
+  finally:
+    shutil.rmtree(local_extensions_dir)
+
+if __name__=="__main__":
+  main()
+