Disabled threaded index-pack for known difficult repositories.

gclient_utils.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generic utils."""
 
 import codecs
 import cStringIO
 import logging
 import os
@@ skipping 12 matching lines @@
 import subprocess2
 
 
 RETRY_MAX = 3
 RETRY_INITIAL_SLEEP = 0.5
 
 
 _WARNINGS = []
 
 
+# These repos are known to cause OOM errors on 32-bit platforms, due the the

[mmoss, 2014/03/25 00:21:44]

It would be nice if this only kicked in under those conditions, but I guess not
a big deal for just this repo.

+# very large objects they contain.  It is not safe to use threaded index-pack
+# when cloning/fetching them.
+THREADED_INDEX_PACK_BLACKLIST = [
+  'https://chromium.googlesource.com/chromium/reference_builds/chrome_win.git'
+]
+
+
 class Error(Exception):
   """gclient exception class."""
   def __init__(self, msg, *args, **kwargs):
     index = getattr(threading.currentThread(), 'index', 0)
     if index:
       msg = '\n'.join('%d> %s' % (index, l) for l in msg.splitlines())
     super(Error, self).__init__(msg, *args, **kwargs)
 
 
 def PrintWarnings():
@@ skipping 940 matching lines @@
 
   The primary constraint is the address space of virtual memory.  The cache
   size limit is per-thread, and 32-bit systems can hit OOM errors if this
   parameter is set too high.
   """
   if platform.architecture()[0].startswith('64'):
     return '2g'
   else:
     return '512m'
 
-def DefaultIndexPackConfig():
+def DefaultIndexPackConfig(url=''):
   """Return reasonable default values for configuring git-index-pack.
 
   Experiments suggest that higher values for pack.threads don't improve
   performance."""
-  return ['-c', 'pack.threads=5', '-c',
-          'core.deltaBaseCacheLimit=%s' % DefaultDeltaBaseCacheLimit()]
+  cache_limit = DefaultDeltaBaseCacheLimit()
+  pack_threads = 1 if url in THREADED_INDEX_PACK_BLACKLIST else 4

[iannucci, 2014/03/24 23:45:10]

Not sure about the hard-coded 4 default. What about:

PACK_THREAD_LIMIT = {
  url: num
}

if url in PACK_THREAD_LIMIT:
  args += ['-c', 'pack.threads=%d' % PACK_THREAD_LIMIT[url]]


?

Then you could put it as 4 for src and blink, 1 for this bad repo, and then let
git figure it out for everything else?

[szager1, 2014/03/24 23:56:44]

I don't have any reason to believe that 4 is more appropriate for src and blink
than for other repos.  In my experiments, 4 was the point of diminishing
returns, but I didn't test it for all deps.

git will just look for a pack.threads config; if it fails to find one, then it
defaults to 3.  So, the only meaningful effect of leaving out the pack.threads
argument on the command-line is to allow users to override it in their
~/.gitconfig.  Which is totally legit for users who are inclined to tinker with
such things.

[mmoss, 2014/03/25 00:21:44]

Yeah, only adding the command-line arg when "necessary" seems like the best bet,
and leave it to default/user value the rest of the time (possibly with "highly
recommended" values coming from etc/gitconfig).

[szager1, 2014/03/25 00:40:25]

OK, the code now only sets pack.threads on the command line for the  blacklisted
repos.

+  return ['-c', 'pack.threads=%d' % pack_threads,
+          '-c', 'core.deltaBaseCacheLimit=%s' % cache_limit]