Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(974)

Side by Side Diff: slave/skia_slave_scripts/check_gs_timestamps.py

Issue 648353002: Remove Skia's forked buildbot code (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master
Patch Set: Address comment Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Checks that the timestamps of gm-actual directories on Google Storage match.
7
8 Checks if the TIMESTAMP_LAST_UPLOAD_STARTED and TIMESTAMP_LAST_UPLOAD_COMPLETED
9 files of all gm-actual directories on Google Storage match. This module can be
10 run from the command-line like this:
11
12 cd buildbot/third_party/chromium_buildbot/slave/\
13 Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release/build/trunk
14
15 PYTHONPATH=../../../../scripts:\
16 ../../../../site_config \
17 python ../../../../../../slave/skia_slave_scripts/check_gs_timestamps.py \
18 --configuration "Debug" --target_platform "" --revision 0 \
19 --make_flags "" --test_args "" --gm_args "" \
20 --bench_args "" --perf_output_basedir "" \
21 --builder_name Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release \
22 --got_revision 0 --is_try False --dest_gsbase gs://rmistry
23
24 """
25
26 import posixpath
27 import sys
28
29 from build_step import BuildStep
30 from utils import old_gs_utils as gs_utils
31 from utils import sync_bucket_subdir
32
33
34 class CheckGoogleStorageTimestamps(BuildStep):
35
36 def _Run(self):
37 dest_gsbase = (self._args.get('dest_gsbase') or
38 sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE)
39
40 # gm-actual directories are of the form:
41 # gs://chromium-skia-gm/playback/gm-actual/platform/builder-name/platform/*
42 # We first get all directories under the first platform dir then get all
43 # directories under the builder-name and finally get the timestamp files
44 # from the final platform dir.
45
46 # Get a list of all platform_dirs.
47 platform_dirs = gs_utils.list_storage_directory(
48 dest_gsbase=dest_gsbase,
49 subdir=posixpath.join(
50 self._storage_playback_dirs.PlaybackRootDir(), 'gm-actual'))
51
52 # Get a list of all platform_builder_dirs.
53 platform_builder_dirs = []
54 for platform_dir in platform_dirs:
55 platform_builder_dirs.extend(gs_utils.list_storage_directory(
56 dest_gsbase=platform_dir,
57 subdir=''))
58 # TODO(rmistry): Ignoring Trybot builders for now. Enable them when they
59 # are more stable. When I ran this script locally they were the only
60 # builders which had differing started and completed timestamps.
61 platform_builder_dirs = filter(
62 lambda x: not x.endswith(posixpath.join('_Trybot', '')),
63 platform_builder_dirs)
64
65 # Get the final list of all platform_builder_platform_dirs.
66 platform_builder_platform_dirs = []
67 for platform_and_builder_dir in platform_builder_dirs:
68 platform_builder_platform_dirs.extend(gs_utils.list_storage_directory(
69 dest_gsbase=platform_and_builder_dir,
70 subdir=''))
71
72 # Check TIMESTAMP_LAST_UPLOAD_STARTED and TIMESTAMP_LAST_UPLOAD_COMPLETED in
73 # each platform_builder_platform directory.
74 failed_gm_actual_dirs = []
75 for timestamp_dir in platform_builder_platform_dirs:
76 gm_actual_started_timestamp = gs_utils.read_timestamp_file(
77 timestamp_file_name=gs_utils.TIMESTAMP_STARTED_FILENAME,
78 gs_base=timestamp_dir,
79 gs_relative_dir='')
80 gm_actual_completed_timestamp = gs_utils.read_timestamp_file(
81 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME,
82 gs_base=timestamp_dir,
83 gs_relative_dir='')
84 if gm_actual_started_timestamp != gm_actual_completed_timestamp:
85 failed_gm_actual_dirs.append(timestamp_dir)
86
87 if failed_gm_actual_dirs:
88 exception_txt = (
89 '\n\nThese are the gm-actual directories with timestamps that do not '
90 'match: %s\n'
91 'This indicates one of two things (can be determined by examining '
92 'the directory):\n'
93 '* The builder is currently running and is in the process of '
94 'updating its gm-actual directory. In this case we do not need to do '
95 'anything.\n'
96 '* The builder\'s gm-actual directory is in an inconsistent state '
97 'and needs to be manually fixed by deleting its '
98 'TIMESTAMP_LAST_UPLOAD_COMPLETED directory.\n\n'
99 % failed_gm_actual_dirs)
100 exception_txt += ('Investigate the suspect directories with the following'
101 ' command(s):\n')
102 for failed_gm_actual_dir in failed_gm_actual_dirs:
103 exception_txt += 'gsutil ls -l %sTIMESTAMP*\n' % failed_gm_actual_dir
104 exception_txt += '\n'
105 raise Exception(exception_txt)
106
107
108 if '__main__' == __name__:
109 sys.exit(BuildStep.RunBuildStep(CheckGoogleStorageTimestamps))
OLDNEW
« no previous file with comments | « slave/skia_slave_scripts/check_compute_engine_disk_usage.sh ('k') | slave/skia_slave_scripts/chrome_canary_update.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698