OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python |
| 2 # Copyright (c) 2008-2010 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 """Generate index.html files for a Google Storage for Developers directory. |
| 7 |
| 8 Google Storage for Developers provides only a raw set of objects. |
| 9 For some buckets we would like to be able to support browsing of the directory |
| 10 tree. This utility will generate the needed index and upload/update it. |
| 11 """ |
| 12 |
| 13 import optparse |
| 14 import posixpath |
| 15 import re |
| 16 import subprocess |
| 17 import sys |
| 18 import tempfile |
| 19 |
| 20 |
| 21 GENERATED_INDEX = '_index.html' |
| 22 |
| 23 |
| 24 def PathToLink(path): |
| 25 return path.replace('gs://', 'https://sandbox.google.com/storage/') |
| 26 |
| 27 |
| 28 def FixupSize(sz): |
| 29 """Convert a size string in bytes to human readable form. |
| 30 |
| 31 Arguments: |
| 32 sz: a size string in bytes |
| 33 Returns: |
| 34 A human readable size in bytes/K/M/G. |
| 35 """ |
| 36 sz = int(sz) |
| 37 if sz < 1000: |
| 38 sz = str(sz) |
| 39 elif sz < 1000000: |
| 40 sz = str(int(sz / 100) / 10.0) + 'K' |
| 41 elif sz < 1000000000: |
| 42 sz = str(int(sz / 100000) / 10.0) + 'M' |
| 43 else: |
| 44 sz = str(int(sz / 100000000) / 10.0) + 'G' |
| 45 return sz |
| 46 |
| 47 |
| 48 def GetPathInfo(path, options): |
| 49 """Collect size, date, md5 for a give gsd path.""" |
| 50 # Check current state. |
| 51 cmd = [options.gsutil, 'ls', '-l', path] |
| 52 p = subprocess.Popen(cmd, stdout=subprocess.PIPE) |
| 53 p_stdout, _ = p.communicate() |
| 54 # Extract intersting fields. |
| 55 fields = {} |
| 56 fields['size'] = FixupSize(re.search('\tObject size:\t([0-9]+)\n', |
| 57 p_stdout).group(1)) |
| 58 fields['md5'] = re.search('\tMD5:\t([^\n]+)\n', p_stdout).group(1) |
| 59 fields['date'] = re.search('\tLast mod:\t([^\n]+)\n', p_stdout).group(1) |
| 60 return fields |
| 61 |
| 62 |
| 63 def GenerateIndex(path, children, directories, options): |
| 64 """Generate index for a given path as needed.""" |
| 65 # Generate index content. |
| 66 index = '' |
| 67 index += '<html>' |
| 68 index += '<head>' |
| 69 index += '<title>Index of %s</title>' % path |
| 70 index += '</head>' |
| 71 index += '<body>' |
| 72 index += '<h1>Index of %s</h1>' % path |
| 73 index += '<table>' |
| 74 index += '<tr>' |
| 75 index += '<th align="left">Name</th>' |
| 76 index += '<th align="left">Last modified</th>' |
| 77 index += '<th align="left">Size</th>' |
| 78 index += '<th align="left">MD5</th>' |
| 79 index += '</tr>' |
| 80 index += '<tr><th colspan="4"><hr></th></tr>' |
| 81 parent = posixpath.dirname(path) |
| 82 if parent != 'gs:': |
| 83 index += '<tr>' |
| 84 index += '<td><a href="%s">Parent Directory</a></td>' % ( |
| 85 PathToLink(posixpath.join(parent, GENERATED_INDEX))) |
| 86 index += '<td> </td>' |
| 87 index += '<td> </td>' |
| 88 index += '<td> </td>' |
| 89 index += '</tr>' |
| 90 for child in children: |
| 91 index += '<tr>' |
| 92 if child in directories: |
| 93 index += '<td><a href="%s">%s</a></td>' % ( |
| 94 PathToLink(posixpath.join(child, GENERATED_INDEX)), |
| 95 posixpath.basename(child)) |
| 96 index += '<td> </td>' |
| 97 index += '<td> </td>' |
| 98 index += '<td> </td>' |
| 99 else: |
| 100 fields = GetPathInfo(child, options) |
| 101 index += '<td><a href="%s">%s</a></td>' % ( |
| 102 PathToLink(child), posixpath.basename(child)) |
| 103 index += '<td>%s</td>' % fields['date'] |
| 104 index += '<td><b>%s</b></td>' % fields['size'] |
| 105 index += '<td>%s</td>' % fields['md5'] |
| 106 index += '</tr>' |
| 107 index += '<tr><th colspan="4"><hr></th></tr>' |
| 108 index += '</table>' |
| 109 index += '</body>' |
| 110 index += '</html>' |
| 111 # Check current state. |
| 112 cmd = [options.gsutil, 'cat', posixpath.join(path, GENERATED_INDEX)] |
| 113 p = subprocess.Popen(cmd, stdout=subprocess.PIPE) |
| 114 p_stdout, _ = p.communicate() |
| 115 # Done if it's alrady right. |
| 116 if p_stdout == index and not options.force: |
| 117 print '%s -- skipping, up to date' % path |
| 118 return |
| 119 # Write to a file. |
| 120 f = tempfile.NamedTemporaryFile(suffix='.html') |
| 121 filename = f.name |
| 122 f.write(index) |
| 123 f.flush() |
| 124 # Upload index. |
| 125 cmd = [options.gsutil, 'cp'] |
| 126 if options.acl: |
| 127 cmd += ['-a', options.acl] |
| 128 cmd += [filename, posixpath.join(path, GENERATED_INDEX)] |
| 129 p = subprocess.Popen(cmd) |
| 130 p.communicate() |
| 131 print '%s -- updated index' % path |
| 132 |
| 133 |
| 134 def GenerateIndexes(path, options): |
| 135 """Generate all relevant indexes for a given gsd path.""" |
| 136 # Get a list of objects under this prefix. |
| 137 p = subprocess.Popen([options.gsutil, 'ls', posixpath.join(path, '*')], |
| 138 stdout=subprocess.PIPE) |
| 139 p_stdout, _ = p.communicate() |
| 140 objects = str(p_stdout).splitlines() |
| 141 objects = [o for o in objects if posixpath.basename(o) != GENERATED_INDEX] |
| 142 # Find common prefixes. |
| 143 directories = set() |
| 144 for o in objects: |
| 145 part = posixpath.dirname(o) |
| 146 while part.startswith(path): |
| 147 directories.add(part) |
| 148 part = posixpath.dirname(part) |
| 149 objects += list(directories) |
| 150 # Generate index for each directory. |
| 151 for d in directories: |
| 152 # Skip directories not on the target path if any. |
| 153 if options.path and not options.path.startswith(d): |
| 154 continue |
| 155 # Find just this directories children. |
| 156 children = [o for o in objects if posixpath.dirname(o) == d] |
| 157 # Generate this directory's index if needed. |
| 158 GenerateIndex(d, children, directories, options) |
| 159 return 0 |
| 160 |
| 161 |
| 162 def main(argv): |
| 163 parser = optparse.OptionParser(usage='usage: %prog [options] gs://base-dir') |
| 164 parser.add_option('-p', '--path', dest='path', |
| 165 help='only update indexes on a given path') |
| 166 parser.add_option('-a', dest='acl', help='acl to set on indexes') |
| 167 parser.add_option('-f', '--force', action='store_true', default=False, |
| 168 dest='force', help='upload all indexes even on match') |
| 169 parser.add_option('', '--gsutil', default='gsutil', |
| 170 dest='gsutil', help='path to gsutil') |
| 171 options, args = parser.parse_args(argv) |
| 172 if len(args) != 2 or not args[1].startswith('gs://'): |
| 173 parser.print_help() |
| 174 return 1 |
| 175 return GenerateIndexes(args[1], options) |
| 176 |
| 177 |
| 178 if __name__ == '__main__': |
| 179 sys.exit(main(sys.argv)) |
OLD | NEW |