Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(118)

Side by Side Diff: slave/skia_slave_scripts/utils/sync_bucket_subdir.py

Issue 648353002: Remove Skia's forked buildbot code (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: Fix launch_slaves, remove more stuff Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Synchronize multiple files with a Google Storage Bucket and subdir.
7 For a lack of a better word 'synchronization' was used, warnings:
8 1) Downloads files that are not found locally
9 2) Uploads files that are not found in the bucket
10 3) Does NOT check version, date or content of the file. If the file is reported
11 to be present both locally and in the bucket, no upload nor download is
12 performed.
13 4) If we find filenames that do not match our expectations, an exception
14 will be raised
15 """
16
17 from slave import slave_utils
18 from utils import download_from_bucket
19 from utils import upload_to_bucket
20
21 import os
22 import posixpath
23 import re
24
25
26 DEFAULT_PERFDATA_GS_BASE = 'gs://chromium-skia-gm'
27 KNOWN_FILENAMES = r'^bench_([0-9a-fr]*)_data.*'
28 IGNORE_UPLOAD_FILENAMES = ('.DS_Store')
29
30
31 def SyncBucketSubdir(directory, dest_gsbase=DEFAULT_PERFDATA_GS_BASE, subdir='',
32 do_upload=True, do_download=True, filenames_filter=KNOWN_FILENAMES,
33 exclude_json=False,
34 min_download_revision=0):
35 """ synchronizes a local directory with a cloud one
36
37 dir: directory to synchronize
38 dest_gsbase: gs:// bucket to synchronize
39 subdir: optional subdirectory within the bucket, multiple directory levels
40 are supported, using Unix relative path syntax ("outer/innner")
41 do_upload: True to perform upload, False otherwise
42 do_download: True to perform download, False otherwise
43 filenames_filter: is a regular expression used to match known file names,
44 and re.search(filenames_filter, file_name).group(1)
45 must return revision number.
46 min_download_revision: don't transfer files whose revision number
47 (based on filenames_filter) is lower than this
48 """
49
50 local_files = set(os.listdir(directory))
51
52 status, output_gsutil_ls = slave_utils.GSUtilListBucket(
53 posixpath.join(dest_gsbase, subdir), [])
54
55 # If there is not at least one file in that subdir, gsutil reports error.
56 # Writing something like GsUtilExistsSubdir is a lot of pain.
57 # We assume that the subdir does not exists, and if there is a real
58 # issue, it will surface later.
59 if status != 0:
60 print 'ls failed'
61 output_gsutil_ls = ''
62
63 output_gsutil_ls = set(output_gsutil_ls.splitlines())
64 gsbase_subdir = posixpath.join(dest_gsbase, subdir, '')
65
66 cloud_files = set()
67 for line in output_gsutil_ls:
68 # Ignore lines with warnings and status messages.
69 if line.startswith(gsbase_subdir) and line != gsbase_subdir:
70 cloud_files.add(line.replace(gsbase_subdir, ''))
71
72 # Download only files not present on the local dir
73 if do_download:
74 to_download = cloud_files.difference(local_files)
75 for file_name in to_download:
76 match = re.search(filenames_filter, file_name)
77 if not match:
78 continue
79
80 # Uploads only files not present on the cloud storage
81 if do_upload:
82 to_upload = local_files.difference(cloud_files)
83 for file_name in to_upload:
84 if file_name not in IGNORE_UPLOAD_FILENAMES:
85 match = re.search(filenames_filter, file_name)
86 if not match or (exclude_json and file_name.endswith('.json')):
87 # Ignore other files, rather than raising an exception
88 continue
89 # Ignore force builds without a revision number.
90 if match.group(1) != '':
91 upload_to_bucket.upload_to_bucket(os.path.join(directory, file_name),
92 dest_gsbase,
93 subdir)
94 return 0
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698