Upload symbols when uploading mojo binaries to the CDN.

mojo/tools/upload_binaries.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Tool to roll Chromium into Mojo. See:
 https://github.com/domokit/mojo/wiki/Rolling-code-between-chromium-and-mojo#chromium---mojo-updates
 """
 
 import argparse
 import glob
+import itertools
 import os
 import subprocess
 import sys
 import tempfile
 import time
 import zipfile
 
 import mopy.gn as gn
 from mopy.config import Config
 from mopy.paths import Paths
 from mopy.version import Version
 
+sys.path.append(os.path.join(os.path.dirname(__file__),
+                             '../../third_party/pyelftools'))

[viettrungluu, 2015/07/10 18:27:01]

os.path.join(os.pardir, os.pardir, "third_party", "pyelftools")

[qsr, 2015/07/15 13:49:11]

Done.

+import elftools.elf.elffile as elffile
+
 BLACKLISTED_APPS = [
   # The network service apps are not produced out of the Mojo repo, but may
   # be present in the build dir.
   "network_service.mojo",
   "network_service_apptests.mojo",
 ]
 
 ARCHITECTURE_INDEPENDENT_FILES = [
   # These are files other than *.mojo files which are part of our binary
   # artifact scheme. These files must be architecture independent.
@@ skipping 38 matching lines @@
 
   depot_tools_path = find_depot_tools.add_depot_tools_to_path()
   gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
 
   if dry_run:
     print str([gsutil_exe, "cp", source, dest])
   else:
     subprocess.check_call([gsutil_exe, "cp", source, dest])
 
 
+def _get_signature(file_object):
+  """Compute a unique signature of a library file.

[viettrungluu, 2015/07/10 18:27:01]

And what if it's not a library file, or even an ELF binary?

[qsr, 2015/07/10 19:35:27]

Was certain it was returning None. Will change.

+
+  TODO(qsr): Move this function to devtools and import it from there to ensure
+  signature computation is always coherent.
+  """
+  elf = elffile.ELFFile(file_object)
+  text_section = elf.get_section_by_name('.text')

[viettrungluu, 2015/07/10 18:27:01]

Apparently, "s are mostly preferred in this file over 's.

[qsr, 2015/07/15 13:49:11]

Done.

+  file_object.seek(text_section['sh_offset'])
+  data = file_object.read(min(4096, text_section['sh_size']))

[viettrungluu, 2015/07/10 18:27:01]

It seems a little weak (in terms of "uniqueness") to look at only 4096 bytes. I
don't see why you don't just hash all of it. (Looking only at .text is a little
weak, but I guess OK for looking at the stack. But maybe you should hash the
entire stripped library (or the .mojo).

[qsr, 2015/07/10 19:35:27]

So this algo is what breakpad use, so I do not see any reason to do anything
else. But the property I really want are:

1) Do not depend on whether the symbol are there or not
2) Computed from a small part of the file. The reason for this is that when
debugging with gdb, I want to be able to compute the hash of a file without
having to transfer it completely over USB because this is slow as hell.

+  def combine((i, c)):
+    return i ^ ord(c)
+  result = 16 * [0]
+  for i in xrange(0, len(data), len(result)):

[viettrungluu, 2015/07/10 18:27:01]

... and I don't understand why you wouldn't use a proper hash function.

[qsr, 2015/07/10 19:35:27]

 Same thing, this is what breakpad is doing and I do not see any reason to
re-invent my own algo.

+    result = map(combine,
+                 itertools.izip_longest(result,
+                                        data[i:i + len(result)],
+                                        fillvalue='\0'))
+  return ''.join(["%02x" % x for x in result])
+
+
+def upload_symbols(config, build_dir, dry_run):
+  dest_dir = "gs://mojo/symbols/"
+  for name in os.listdir(build_dir):
+    path = os.path.join(build_dir, name)
+    if (os.path.isfile(path) and
+        not os.path.islink(path) and
+        os.access(path, os.X_OK) and

[viettrungluu, 2015/07/10 18:27:01]

This seems to assume that all files in the build dir with the x bit set are
interesting libraries, which seems wrong to me:

* It's not clear to me why we set the x bit for .so's in the first place, so if
we ever changed that this would break.
* This will also include tools, such as md5sum_bin.
* This will even include tools built for the host, such as md5sum_bin_host.

So for .mojo's, we should modify the rule to copy (or maybe symlink) the
unstripped library somewhere, so that we'll have a canonical(ish[*]) set of
things to upload. (We can deal with the shell itself separately.)

[*] Of course, if we're not doing clobber builds, then we could have stale
stuff.

[qsr, 2015/07/10 19:35:27]

 Getting some more artifact is not really a problem.
 Stale stuff is not a problem either. We would just re-upload old symbols.

 Will change the mojo application target to hard link to a new directory. I
don't want symlinks because stale symlink without their target might break this
script.

+        not path.endswith('.mojo')):
+      with open(path) as f:
+        dest = dest_dir + _get_signature(f)
+        upload(config, path, dest, dry_run)
+
+
 def upload_shell(config, dry_run, verbose):
   paths = Paths(config)
   zipfile_name = target(config)
   version = Version().version
 
   # Upload the binary.
   # TODO(blundell): Change this to be in the same structure as the LATEST files,
   # e.g., gs://mojo/shell/linux-x64/<version>/shell.zip.
   dest = "gs://mojo/shell/" + version + "/" + zipfile_name + ".zip"
   with tempfile.NamedTemporaryFile() as zip_file:
@@ skipping 80 matching lines @@
 
   build_directory = gn.BuildDirectoryForConfig(config, Paths(config).src_root)
   apps_to_upload = find_apps_to_upload(build_directory)
   for app in apps_to_upload:
     upload_app(app, config, args.dry_run)
 
   files_to_upload = find_architecture_independent_files(build_directory)
   for file_to_upload in files_to_upload:
     upload_file(file_to_upload, config, args.dry_run)
 
+  upload_symbols(config, build_directory, args.dry_run)
+
   return 0
 
 if __name__ == "__main__":
   sys.exit(main())