| OLD | NEW |
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Synchronize multiple files with a Google Storage Bucket and subdir. | |
| 7 For a lack of a better word 'synchronization' was used, warnings: | |
| 8 1) Downloads files that are not found locally | |
| 9 2) Uploads files that are not found in the bucket | |
| 10 3) Does NOT check version, date or content of the file. If the file is reported | |
| 11 to be present both locally and in the bucket, no upload nor download is | |
| 12 performed. | |
| 13 4) If we find filenames that do not match our expectations, an exception | |
| 14 will be raised | |
| 15 """ | |
| 16 | |
| 17 from slave import slave_utils | |
| 18 from utils import download_from_bucket | |
| 19 from utils import upload_to_bucket | |
| 20 | |
| 21 import os | |
| 22 import posixpath | |
| 23 import re | |
| 24 | |
| 25 | |
| 26 DEFAULT_PERFDATA_GS_BASE = 'gs://chromium-skia-gm' | |
| 27 KNOWN_FILENAMES = r'^bench_([0-9a-fr]*)_data.*' | |
| 28 IGNORE_UPLOAD_FILENAMES = ('.DS_Store') | |
| 29 | |
| 30 | |
| 31 def SyncBucketSubdir(directory, dest_gsbase=DEFAULT_PERFDATA_GS_BASE, subdir='', | |
| 32 do_upload=True, do_download=True, filenames_filter=KNOWN_FILENAMES, | |
| 33 exclude_json=False, | |
| 34 min_download_revision=0): | |
| 35 """ synchronizes a local directory with a cloud one | |
| 36 | |
| 37 dir: directory to synchronize | |
| 38 dest_gsbase: gs:// bucket to synchronize | |
| 39 subdir: optional subdirectory within the bucket, multiple directory levels | |
| 40 are supported, using Unix relative path syntax ("outer/innner") | |
| 41 do_upload: True to perform upload, False otherwise | |
| 42 do_download: True to perform download, False otherwise | |
| 43 filenames_filter: is a regular expression used to match known file names, | |
| 44 and re.search(filenames_filter, file_name).group(1) | |
| 45 must return revision number. | |
| 46 min_download_revision: don't transfer files whose revision number | |
| 47 (based on filenames_filter) is lower than this | |
| 48 """ | |
| 49 | |
| 50 local_files = set(os.listdir(directory)) | |
| 51 | |
| 52 status, output_gsutil_ls = slave_utils.GSUtilListBucket( | |
| 53 posixpath.join(dest_gsbase, subdir), []) | |
| 54 | |
| 55 # If there is not at least one file in that subdir, gsutil reports error. | |
| 56 # Writing something like GsUtilExistsSubdir is a lot of pain. | |
| 57 # We assume that the subdir does not exists, and if there is a real | |
| 58 # issue, it will surface later. | |
| 59 if status != 0: | |
| 60 print 'ls failed' | |
| 61 output_gsutil_ls = '' | |
| 62 | |
| 63 output_gsutil_ls = set(output_gsutil_ls.splitlines()) | |
| 64 gsbase_subdir = posixpath.join(dest_gsbase, subdir, '') | |
| 65 | |
| 66 cloud_files = set() | |
| 67 for line in output_gsutil_ls: | |
| 68 # Ignore lines with warnings and status messages. | |
| 69 if line.startswith(gsbase_subdir) and line != gsbase_subdir: | |
| 70 cloud_files.add(line.replace(gsbase_subdir, '')) | |
| 71 | |
| 72 # Download only files not present on the local dir | |
| 73 if do_download: | |
| 74 to_download = cloud_files.difference(local_files) | |
| 75 for file_name in to_download: | |
| 76 match = re.search(filenames_filter, file_name) | |
| 77 if not match: | |
| 78 continue | |
| 79 | |
| 80 # Uploads only files not present on the cloud storage | |
| 81 if do_upload: | |
| 82 to_upload = local_files.difference(cloud_files) | |
| 83 for file_name in to_upload: | |
| 84 if file_name not in IGNORE_UPLOAD_FILENAMES: | |
| 85 match = re.search(filenames_filter, file_name) | |
| 86 if not match or (exclude_json and file_name.endswith('.json')): | |
| 87 # Ignore other files, rather than raising an exception | |
| 88 continue | |
| 89 # Ignore force builds without a revision number. | |
| 90 if match.group(1) != '': | |
| 91 upload_to_bucket.upload_to_bucket(os.path.join(directory, file_name), | |
| 92 dest_gsbase, | |
| 93 subdir) | |
| 94 return 0 | |
| OLD | NEW |