Generalize upload_network_service.py to support uploading other services.

mojo/services/upload_service.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import argparse
 import imp
 import os
 import subprocess
 import sys
 import tempfile
 import time
 import zipfile
 
+# A service's name is defined as the name of its subdirectory in the directory
+# containing this file.

[ppi, 2015/02/03 16:08:09]

nit: note that it also defines the gs path that the binary will be uploaded to.

[blundell, 2015/02/03 16:19:40]

I actually would prefer not to do this unless you feel strongly as I fear that
the comment could easily grow out-of-date if we decide to change the format of
the bucketing.

+SERVICES = [ "network", "html_viewer" ]
+
+SERVICE_BINARY_NAMES = {
+    "network" : "network_service.mojo",
+    "html_viewer" : "html_viewer.mojo"
+}
+
+# The network service is downloaded out-of-band rather than dynamically by the
+# shell and thus can be stored zipped in the cloud. Other services are intended
+# to be downloaded dynamically by the shell, which doesn't currently understand
+# zipped binaries.
+SERVICES_WITH_ZIPPED_BINARIES = [ "network" ]
+
 if not sys.platform.startswith("linux"):
   print "Only support linux for now"
   sys.exit(1)
 
 root_path = os.path.realpath(
     os.path.join(
         os.path.dirname(
             os.path.realpath(__file__)),
         os.pardir,
-        os.pardir,
         os.pardir))
 version = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root_path)
 version = version.strip()
 
 find_depot_tools_path = os.path.join(root_path, "tools", "find_depot_tools.py")
 find_depot_tools = imp.load_source("find_depot_tools", find_depot_tools_path)
 
 depot_tools_path = find_depot_tools.add_depot_tools_to_path()
 gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
 
 
 def gsutil_cp(source, dest, dry_run):
   if dry_run:
     print "gsutil cp %s %s" % (source, dest)
   else:
     subprocess.check_call([gsutil_exe, "cp", source, dest])
 
 
-def upload_mojoms(dry_run):
+def upload_mojoms(service, dry_run):
+  script_dir = os.path.dirname(os.path.realpath(__file__))
+  service_dir = os.path.join(script_dir, service)
   absolute_mojom_directory_path = os.path.join(
-      os.path.dirname(os.path.realpath(__file__)),
+      service_dir,
       "public",
       "interfaces")
-  dest = "gs://mojo/network/" + version + "/" + "mojoms.zip"
+
+  if not os.path.exists(absolute_mojom_directory_path):
+    # This service has no interfaces.
+    return
+
+  dest = "gs://mojo/" + service + "/" + version + "/" + "mojoms.zip"
   with tempfile.NamedTemporaryFile() as mojom_zip_file:
     with zipfile.ZipFile(mojom_zip_file, 'w') as z:
       for root, _, files in os.walk(absolute_mojom_directory_path):
         for filename in files:
           absolute_file_path = os.path.join(root, filename)
           relative_file_path = os.path.relpath(absolute_file_path, root)
           z.write(absolute_file_path, relative_file_path)
     gsutil_cp(mojom_zip_file.name, dest, dry_run)
 
 
-def upload_binary(binary_path, platform, dry_run):
-  absolute_binary_path = os.path.join(root_path, binary_path)
-  binary_dest = "gs://mojo/network/" + version + "/" + platform + ".zip"
+def upload_binary(service, binary_dir, platform, dry_run):
+  binary_name = SERVICE_BINARY_NAMES[service]
+  absolute_binary_path = os.path.join(root_path, binary_dir, binary_name)
+  binary_dest_prefix = "gs://mojo/" + service + "/" + version + "/" + platform
+
+  if service not in SERVICES_WITH_ZIPPED_BINARIES:
+    binary_dest = binary_dest_prefix + "/" + binary_name
+    gsutil_cp(absolute_binary_path, binary_dest, dry_run)
+    return
+
+  # Zip the binary before uploading it to the cloud.
+  binary_dest = binary_dest_prefix + ".zip"
   with tempfile.NamedTemporaryFile() as binary_zip_file:
     with zipfile.ZipFile(binary_zip_file, 'w') as z:
       with open(absolute_binary_path) as service_binary:
-        zipinfo = zipfile.ZipInfo("network_service.mojo")
+        zipinfo = zipfile.ZipInfo(binary_name)
         zipinfo.external_attr = 0o777 << 16
         zipinfo.compress_type = zipfile.ZIP_DEFLATED
         zipinfo.date_time = time.gmtime(os.path.getmtime(absolute_binary_path))
         z.writestr(zipinfo, service_binary.read())
     gsutil_cp(binary_zip_file.name, binary_dest, dry_run)
 
 
 def main():
   parser = argparse.ArgumentParser(
-      description="Upload network service mojoms and binaries to Google " +
-                  "storage")
+      description="Upload service mojoms and binaries to Google storage")
   parser.add_argument("-n", "--dry-run", action="store_true", help="Dry run")
   parser.add_argument(
-      "--linux-x64-binary-path",
-      help="Path to the linux-x64 network service binary relative to the " +
-           "repo root, e.g. out/Release/network_service.mojo")
+      "--linux-x64-binary-dir",
+      help="Path to the dir containing the linux-x64 service binary relative "
+           "to the repo root, e.g. out/Release")
   parser.add_argument(
-      "--android-arm-binary-path",
-      help="Path to the android-arm network service binary relative to the " +
-           "repo root, e.g. out/android_Release/network_service.mojo")
+      "--android-arm-binary-dir",
+      help="Path to the dir containing the android-arm service binary relative "
+           "to the repo root, e.g. out/android_Release")
+  parser.add_argument("service",

[ppi, 2015/02/03 16:08:09]

The mandatory argument should go first I think.

[ppi, 2015/02/03 16:18:20]

As discussed live, turn's out it shouldn't, please disregard. Thanks.

+                      help="The service to be uploaded (one of %s)" % SERVICES)
 
   args = parser.parse_args()
-  upload_mojoms(args.dry_run)
-  if args.linux_x64_binary_path:
-    upload_binary(args.linux_x64_binary_path, "linux-x64", args.dry_run)
-  if args.android_arm_binary_path:
-    upload_binary(args.android_arm_binary_path, "android-arm", args.dry_run)
+
+  if args.service not in SERVICES:
+    print args.service + " is not one of the recognized services:"
+    print SERVICES
+    sys.exit(1)

[jamesr, 2015/02/03 18:29:57]

return 1, don't sys.exit().

[blundell, 2015/02/04 10:04:18]

Done.

+
+  upload_mojoms(args.service, args.dry_run)
+  if args.linux_x64_binary_dir:
+    upload_binary(args.service, args.linux_x64_binary_dir,
+                  "linux-x64", args.dry_run)
+  if args.android_arm_binary_dir:
+    upload_binary(args.service, args.android_arm_binary_dir,
+                  "android-arm", args.dry_run)
 
   if not args.dry_run:
     print "Uploaded artifacts for version %s" % (version, )
   else:
     print "No artifacts uploaded (dry run)"
   return 0
 
 if __name__ == '__main__':
   sys.exit(main())