Extension profile generator + benchmark for startup with profile.

tools/perf/profile_creators/update_remote_extensions.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import csv
+import os
+import optparse

[erikchen, 2015/07/14 23:36:15]

alphabetize

[sydli, 2015/07/15 22:56:24]

Done.

+import shutil
+import sys
+import tempfile
+import urllib2
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+    'telemetry'))
+
+from catapult_base import cloud_storage
+
+
+# Google Cloud Storage bucket to upload to
+BUCKET_NAME = "chrome-partner-telemetry"

[erikchen, 2015/07/14 23:36:15]

no need for this variable

[sydli, 2015/07/15 22:56:24]

Done.

+DEFAULT_REMOTE_DIR = "extension_set"
+
+def DownloadCRXFromCWS(ext_id, dst):
+  """ Download CRX (Chrome extension file) specified by extension_id from

[erikchen, 2015/07/14 23:36:15]

Please see python style guide for formatting of this comment.

"""This must be a 1-line summary.

Long description goes here.
Can span multiple lines"""

[sydli, 2015/07/15 22:56:24]

Done.

+  Chrome Web Store, into file specified by dst. """
+  dst_path = os.path.join(dst, "%s.crx" % ext_id)
+  cws_url = ("https://clients2.google.com/service/update2/crx?response="
+          "redirect&prodversion=38.0&x=id%%3D%s%%26installsource%%3D"
+          "ondemand%%26uc" % ext_id)
+  response = urllib2.urlopen(cws_url)
+  if response.getcode() is not 200:
+    return None

[erikchen, 2015/07/14 23:36:15]

This seems like a silent failure. I would throw an exception and allow the
exception to bubble to the top. Or you could add handling for the return value
of this function. Either way, please document in function comment.

[sydli, 2015/07/15 22:56:24]

Documented return value and will handle in caller (prints warning message)

+  with open(dst_path, "w") as f:
+    f.write(response.read())
+  return dst_path
+
+def UpdateExtensionsInCloud(local_extensions_dir, extensions_csv, remote_dir):
+  """ From well-formatted CSV file containing some set of extensions
+  (extensions_csv), download them into local temp directory and update
+  the remote directory. """
+  # Download CRX to temp files
+  with open(extensions_csv, 'rb') as csvfile:
+    reader = csv.reader(csvfile)
+    reader.next() # skip header line
+    for row in reader:
+      extension_id = row[1]
+      print "Fetching extension %s..." % extension_id
+      DownloadCRXFromCWS(extension_id, local_extensions_dir)
+    # Delete remote directory if it exists

[erikchen, 2015/07/14 23:36:15]

You are deleting, and then uploading. This set of operations isn't atomic.

What if all of the files are zipped first, and then you just upload the new zip
to overwrite the old zip? I believe that cloud storage has guarantees about the
atomicity of this operation. aiolos@ would know more.

[sydli, 2015/07/15 22:56:24]

Oh, my bad; there's also toctou problem. Will archive files and use a single
operation.

+  if cloud_storage.Exists(BUCKET_NAME, remote_dir):
+    cloud_storage.Delete(BUCKET_NAME, remote_dir, recursive=True)
+  print "Uploading extensions to cloud..."
+  cloud_storage.Insert(BUCKET_NAME, remote_dir,
+          local_extensions_dir, recursive=True)
+
+def GetCSVFromArgs():
+  parser = optparse.OptionParser()
+  parser.add_option("-e", "--extension-csv", dest="extension_csv",
+          help="CSV of extensions to load.")
+  (options, _) = parser.parse_args()
+  if not options.extension_csv:
+    parser.error("Must specify --extension-csv option.")
+  return options.extension_csv
+
+def main():
+  # TODO(sydli) check integrity of CSV
+  extension_csv = GetCSVFromArgs()
+  local_extensions_dir = tempfile.mkdtemp()
+  UpdateExtensionsInCloud(local_extensions_dir,
+          extension_csv, DEFAULT_REMOTE_DIR)
+  # Clean up
+  shutil.rmtree(local_extensions_dir)

[erikchen, 2015/07/14 23:36:15]

You probably want to wrap UpdateExtensionsInCloud() in a try/finally so that
rmtree is always run afterwords.

[sydli, 2015/07/15 22:56:24]

Done.

+
+if __name__=="__main__":
+  main()
+