Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(563)

Unified Diff: slave/skia_slave_scripts/utils/sync_bucket_subdir.py

Issue 648353002: Remove Skia's forked buildbot code (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: Address comment Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: slave/skia_slave_scripts/utils/sync_bucket_subdir.py
diff --git a/slave/skia_slave_scripts/utils/sync_bucket_subdir.py b/slave/skia_slave_scripts/utils/sync_bucket_subdir.py
deleted file mode 100755
index 8c2325ee1490189fff4bf33f2dc91cb448d422f5..0000000000000000000000000000000000000000
--- a/slave/skia_slave_scripts/utils/sync_bucket_subdir.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Synchronize multiple files with a Google Storage Bucket and subdir.
-For a lack of a better word 'synchronization' was used, warnings:
-1) Downloads files that are not found locally
-2) Uploads files that are not found in the bucket
-3) Does NOT check version, date or content of the file. If the file is reported
- to be present both locally and in the bucket, no upload nor download is
- performed.
-4) If we find filenames that do not match our expectations, an exception
- will be raised
-"""
-
-from slave import slave_utils
-from utils import download_from_bucket
-from utils import upload_to_bucket
-
-import os
-import posixpath
-import re
-
-
-DEFAULT_PERFDATA_GS_BASE = 'gs://chromium-skia-gm'
-KNOWN_FILENAMES = r'^bench_([0-9a-fr]*)_data.*'
-IGNORE_UPLOAD_FILENAMES = ('.DS_Store')
-
-
-def SyncBucketSubdir(directory, dest_gsbase=DEFAULT_PERFDATA_GS_BASE, subdir='',
- do_upload=True, do_download=True, filenames_filter=KNOWN_FILENAMES,
- exclude_json=False,
- min_download_revision=0):
- """ synchronizes a local directory with a cloud one
-
- dir: directory to synchronize
- dest_gsbase: gs:// bucket to synchronize
- subdir: optional subdirectory within the bucket, multiple directory levels
- are supported, using Unix relative path syntax ("outer/innner")
- do_upload: True to perform upload, False otherwise
- do_download: True to perform download, False otherwise
- filenames_filter: is a regular expression used to match known file names,
- and re.search(filenames_filter, file_name).group(1)
- must return revision number.
- min_download_revision: don't transfer files whose revision number
- (based on filenames_filter) is lower than this
- """
-
- local_files = set(os.listdir(directory))
-
- status, output_gsutil_ls = slave_utils.GSUtilListBucket(
- posixpath.join(dest_gsbase, subdir), [])
-
- # If there is not at least one file in that subdir, gsutil reports error.
- # Writing something like GsUtilExistsSubdir is a lot of pain.
- # We assume that the subdir does not exists, and if there is a real
- # issue, it will surface later.
- if status != 0:
- print 'ls failed'
- output_gsutil_ls = ''
-
- output_gsutil_ls = set(output_gsutil_ls.splitlines())
- gsbase_subdir = posixpath.join(dest_gsbase, subdir, '')
-
- cloud_files = set()
- for line in output_gsutil_ls:
- # Ignore lines with warnings and status messages.
- if line.startswith(gsbase_subdir) and line != gsbase_subdir:
- cloud_files.add(line.replace(gsbase_subdir, ''))
-
- # Download only files not present on the local dir
- if do_download:
- to_download = cloud_files.difference(local_files)
- for file_name in to_download:
- match = re.search(filenames_filter, file_name)
- if not match:
- continue
-
- # Uploads only files not present on the cloud storage
- if do_upload:
- to_upload = local_files.difference(cloud_files)
- for file_name in to_upload:
- if file_name not in IGNORE_UPLOAD_FILENAMES:
- match = re.search(filenames_filter, file_name)
- if not match or (exclude_json and file_name.endswith('.json')):
- # Ignore other files, rather than raising an exception
- continue
- # Ignore force builds without a revision number.
- if match.group(1) != '':
- upload_to_bucket.upload_to_bucket(os.path.join(directory, file_name),
- dest_gsbase,
- subdir)
- return 0
« no previous file with comments | « slave/skia_slave_scripts/utils/old_gs_utils_test.py ('k') | slave/skia_slave_scripts/utils/sync_skia_in_chrome.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698