git-sync-deps tool

tools/grab_deps.py

+#!/usr/bin/python
+# Copyright 2014 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import sys
+import os
+import subprocess
+import git_utils
+import subprocess
+import misc_utils
+
+
+deps = {

[borenet, 2014/03/10 18:36:12]

We should definitely be reading the DEPS file instead of duplicating the list
here.

[hal.canary, 2014/03/10 20:03:08]

Why write a parser?

[epoger, 2014/03/10 20:11:39]

Because certain important users of Skia use DEPS files, and it may be useful for
their DEPS files to reference ours.

[borenet, 2014/03/10 20:13:30]

No parser is needed; we can do something like:
environment = {}
exectfile('DEPS', environment)
deps = environment['solutions']

I think this is very worthwhile since it will keep the gclient workflow working,
at least in the short term.  That will make things much easier on the bots.  If
we do want to switch all at once, we should remove the DEPS file altogether, but
again, I don't think that's the right thing to do right now.

+  "ALL" : {
+    "third_party/externals/depot_tools" : (

[tfarina, 2014/03/10 19:00:21]

Then why we need an 'extra' copy of depot_tools. Was it solely for ninja? If it
was, then why not let the dev install ninja standalone? You certainly don't need
for that.

+      'https://chromium.googlesource.com/chromium/tools/depot_tools.git',
+      None # None means 'track origin/master'
+    ),
+    "third_party/externals/angle" : (
+      "https://chromium.googlesource.com/external/angleproject.git",
+      None
+    ),
+    "third_party/externals/fontconfig" : (
+      "https://skia.googlesource.com/third_party/fontconfig.git",
+      "2.11.0"
+    ),
+    "third_party/externals/freetype" : (
+      "https://skia.googlesource.com/third_party/freetype2.git",
+      "VER-2-5-0-1"
+    ),
+    "third_party/externals/gyp" : (
+      "https://chromium.googlesource.com/external/gyp.git",
+      "5917c6a6b77c9e97a0cbb66847194381bd36ec4c"
+    ),
+    "third_party/externals/iconv" : (
+      "https://skia.googlesource.com/third_party/libiconv.git",
+      "v1.14"
+    ),
+    "third_party/externals/libjpeg" : (
+      "https://chromium.googlesource.com/chromium/deps/libjpeg_turbo.git",
+      "82ce8a6d4ebe12a177c0c3597192f2b4f09e81c3"
+    ),
+    "third_party/externals/jsoncpp" : (
+      "https://chromium.googlesource.com/external/jsoncpp/jsoncpp.git",
+      "ab1e40f3bce061ea6f9bdc60351d6cde2a4f872b",
+    ),
+    "third_party/externals/jsoncpp-chromium" : (
+      "https://chromium.googlesource.com/chromium/src/third_party/jsoncpp.git",
+      "41239939c0c60481f34887d52c038facf05f5533"
+    ),
+    "third_party/externals/libwebp" : (
+      "https://chromium.googlesource.com/webm/libwebp.git",
+      "3fe91635df8734b23f3c1b9d1f0c4fa8cfaf4e39"
+    ),
+    "third_party/externals/poppler" : (
+      "https://skia.googlesource.com/third_party/poppler.git",
+      "poppler-0.22.5",
+    ),
+  },
+  "android" : {
+    "platform_tools/android/third_party/externals/expat" : (
+      "https://android.googlesource.com/platform/external/expat.git",
+      "android-4.2.2_r1.2",
+    ),
+    "platform_tools/android/third_party/externals/gif" : (
+      "https://android.googlesource.com/platform/external/giflib.git",
+      "android-4.2.2_r1.2",
+    ),
+    "platform_tools/android/third_party/externals/png" : (
+      "https://android.googlesource.com/platform/external/libpng.git",
+      "android-4.2.2_r1.2",
+    ),
+    "platform_tools/android/third_party/externals/jpeg" : (
+      "https://android.googlesource.com/platform/external/jpeg.git",
+      "android-4.2.2_r1.2",
+    ),
+  },
+  "chromeos" : {
+    "platform_tools/chromeos/third_party/externals/gif" : (
+      "https://android.googlesource.com/platform/external/giflib.git",
+      "android-4.2.2_r1.2",
+    ),
+    "platform_tools/chromeos/toolchain/src/third_party/chromite" : (
+      "https://chromium.googlesource.com/chromiumos/chromite.git",
+      "d6a4c7e0ee4d53ddc5238dbddfc0417796a70e54",
+    ),
+    "platform_tools/chromeos/toolchain/src/third_party/pyelftools" : (
+      "https://chromium.googlesource.com/chromiumos/third_party/pyelftools.git",
+      "bdc1d380acd88d4bfaf47265008091483b0d614e",
+    ),
+  },
+  # barelinux is a DEPS target that has no shared libraries to link
+  # to, similar to android or chromeos.
+  "barelinux" : {
+    "third_party/externals/giflib" : (
+      "https://android.googlesource.com/platform/external/giflib.git",
+      "android-4.2.2_r1.2",
+    ),
+    "third_party/externals/libpng" : (
+      "https://android.googlesource.com/platform/external/libpng.git",
+      "android-4.2.2_r1.2",
+    ),
+    "third_party/externals/zlib" : (
+      "https://android.googlesource.com/platform/external/zlib.git",
+      "android-4.2.2_r1.2",
+    ),
+  }
+}
+
+
+def git_checkout_to_directory(repo, checkout, directory):
+  """TODO(halcanary): document this function.
+  """
+  git = git_utils.git_executable()
+  assert git
+  git_path = os.path.join(directory, '.git')
+  if os.path.exists(directory) and not os.path.isdir(git_path):
+    if os.path.isdir(directory) and not os.path.islink(directory):
+      shutil.rmtree(directory)
+    elif os.path.isdir(directory) and not os.path.islink(directory):
+      os.remove(directory)
+  if not os.path.isdir(directory):
+    subprocess.check_call([git, 'clone', repo, directory])
+  subprocess.check_call([git, 'fetch'])
+  with misc_utils.ChangeDir(directory):
+    if not checkout:
+      checkout = 'origin/master'
+    print '>>>>', repo
+    print '>>>>', checkout
+    subprocess.check_call([git, 'checkout', '--quiet', checkout])
+    print

[borenet, 2014/03/10 18:36:12]

I'd like to simplify this logic.  I think we can make the assumption that nobody
is going to modify the third party deps in place.

try:
  subprocess.check_call([git, 'fetch'])
  subprocess.check_call([git, 'reset', '--hard', hash])
except:
  shutil.rmtree(directory)
if not os.path.exists(directory):
  subprocess.check_call([git, 'clone', repo])
  subprocess.check_call([git, 'reset', '--hard', hash])

[hal.canary, 2014/03/10 20:03:08]

Your logic is fine, but I prefer git-checkout to git-reset-hard.

[borenet, 2014/03/10 20:13:30]

Why?  Just curious.  IMO reset is nice because it keeps us on the master branch
and enforces that there are no local changes.

+
+
+def main(argv):
+  dependencies = deps["ALL"].copy()
+  for arg in argv:
+    if arg not in deps:
+      print arg, '?'
+      exit(1)
+    for dep in deps[arg]:
+      dependencies[dep] = deps[arg][dep]
+  for directory in dependencies:
+    repo, checkout = dependencies[directory]
+    git_checkout_to_directory(repo, checkout, directory)
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])
+