Add URL recipe module from "depot_tools".

recipe_modules/url/resources/pycurl.py

+#!/usr/bin/env python
+# Copyright 2017 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+# NOTE: This was imported from Chromium's "tools/build" at revision:
+# 65976b6e2a612439681dc42830e90dbcdf550f40
+
+import argparse
+import json
+import logging
+import os
+import sys
+import time
+
+import requests
+import requests.adapters
+import requests.models
+from requests.packages.urllib3.util.retry import Retry
+
+
+class Output(object):
+  """Output is a file-like object which writes content to a sink.
+
+  If a prefix is supplied, it will validate and discard that prefix before
+  writing output. If the prefix did not validate, a ValueError will be raised.
+  """
+
+  def __init__(self, sink, prefix=None):
+    self.total = 0
+
+    self._sink = sink
+    self._prefix = prefix
+    self._prefix_idx = 0
+    self._prefix_buf = []
+
+  def write(self, content):

[iannucci, 2017/05/12 00:53:52]

assert len(prefix) < CHUNK_SIZE, use iter() interface

[dnj, 2017/05/12 02:15:04]

Done.

+    self.total += len(content)
+
+    # If we still have prefix remaining, read and validate it.
+    if self._prefix and self._prefix_idx < len(self._prefix):
+      prefix_part = self._prefix[self._prefix_idx:][:len(content)]
+      d = content[:len(prefix_part)]
+      self._prefix_buf.append(d)
+      self._prefix_idx += len(prefix_part)
+
+      if d != prefix_part:
+        total_prefix = ''.join(self._prefix_buf)
+        raise ValueError(
+            'Expected prefix was not observed: [%s] != [%s]...' % (
+              total_prefix, self._prefix[:len(total_prefix)]))
+      content = content[len(prefix_part):]
+
+    if content:
+      self._sink.write(content)
+
+
+def _download(url, outfile, headers, transient_retry, strip_prefix):
+  s = requests.Session()
+  if transient_retry:
+    # See http://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html
+    retry = Retry(
+      total=10,
+      connect=5,
+      read=5,
+      redirect=5,
+      status_forcelist=range(500, 600),
+      backoff_factor=0.2,
+    )
+    print retry
+    s.mount(url, requests.adapters.HTTPAdapter(max_retries=retry))
+
+
+  logging.info('Connecting...')
+  r = s.get(url, headers=headers, stream=True)
+  if r.status_code != requests.codes.ok:
+    r.raise_for_status()
+
+  if outfile:
+    fd = open(outfile, 'wb')
+  else:
+    fd = sys.stdout
+  with fd:
+    out = Output(fd, prefix=strip_prefix)
+    logging.info('Downloading...')
+    for chunk in r.iter_content(1024*1024):
+      out.write(chunk)
+      logging.info('Downloaded %.1f MB so far', out.total / 1024 / 1024)
+  return r.status_code, out.total
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description='Get a url and print its document.',
+      prog='./runit.py pycurl.py')
+  parser.add_argument('url', help='the url to fetch')

[iannucci, 2017/05/12 00:53:52]

urlparse validate (up in the recipe)?

[dnj, 2017/05/12 02:15:04]

Done.

+  parser.add_argument('--status-json', metavar='PATH', action='store',

[iannucci, 2017/05/12 00:53:52]

required=True ?

[dnj, 2017/05/12 02:15:04]

Done.

+      help='Write HTTP status result JSON. If set, all complete HTTP '
+           'responses will exit with 0, regardless of their status code.')
+  parser.add_argument('--no-transient-retry', action='store_true',
+      help='Do not perform automatic retries on transient failures.')
+  parser.add_argument('--headers-json', action='store',
+      help='A json file containing any headers to include with the request.')
+  parser.add_argument('--outfile', help='write output to this file')
+  parser.add_argument('--strip-prefix', action='store',
+      help='Expect this string at the beginning of the response, and strip it.')
+
+  args = parser.parse_args()
+
+  headers = None
+  if args.headers_json:
+    with open(args.headers_json, 'r') as json_file:
+      headers = json.load(json_file)
+
+  status = {}
+  try:
+    status_code, size = _download(
+        args.url, args.outfile, headers, not args.no_transient_retry,
+        args.strip_prefix)
+    status = {
+      'status_code': status_code,
+      'success': True,
+      'size': size,
+    }
+  except requests.HTTPError as e:
+    if not args.status_json:
+      raise
+    status = {
+      'status_code': e.response.status_code,
+      'success': False,
+    }
+
+  if args.status_json:
+    with open(args.status_json, 'w') as fd:
+      json.dump(status, fd)
+  return 0
+
+
+if __name__ == '__main__':
+  logging.basicConfig()
+  logging.getLogger().setLevel(logging.INFO)
+  logging.getLogger("requests").setLevel(logging.DEBUG)
+  sys.exit(main())
+
+
+##
+# The following section is read by "vpython" and used to construct the
+# VirtualEnv for this tool.
+#
+# These imports were lifted from "/bootstrap/venv.cfg".
+##
+# [VPYTHON:BEGIN]
+#
+# wheel: <
+#   name: "infra/python/wheels/cryptography/${platform}_${py_version}_${py_abi}"
+#   version: "version:1.8.1"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/appdirs-py2_py3"
+#   version: "version:1.4.3"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/asn1crypto-py2_py3"
+#   version: "version:0.22.0"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/enum34-py2"
+#   version: "version:1.1.6"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/cffi/${platform}_${py_version}_${py_abi}"
+#   version: "version:1.10.0"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/idna-py2_py3"
+#   version: "version:2.5"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/ipaddress-py2"
+#   version: "version:1.0.18"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/packaging-py2_py3"
+#   version: "version:16.8"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/pyasn1-py2_py3"
+#   version: "version:0.2.3"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/pycparser-py2_py3"
+#   version: "version:2.17"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/pyopenssl-py2_py3"
+#   version: "version:17.0.0"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/pyparsing-py2_py3"
+#   version: "version:2.2.0"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/setuptools-py2_py3"
+#   version: "version:34.3.2"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/six-py2_py3"
+#   version: "version:1.10.0"
+# >
+#
+# wheel: <
+#   name: "infra/python/wheels/requests-py2_py3"
+#   version: "version:2.13.0"
+# >
+#
+# [VPYTHON:END]
+##