OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """Checks that the timestamps of gm-actual directories on Google Storage match. | |
7 | |
8 Checks if the TIMESTAMP_LAST_UPLOAD_STARTED and TIMESTAMP_LAST_UPLOAD_COMPLETED | |
9 files of all gm-actual directories on Google Storage match. This module can be | |
10 run from the command-line like this: | |
11 | |
12 cd buildbot/third_party/chromium_buildbot/slave/\ | |
13 Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release/build/trunk | |
14 | |
15 PYTHONPATH=../../../../scripts:\ | |
16 ../../../../site_config \ | |
17 python ../../../../../../slave/skia_slave_scripts/check_gs_timestamps.py \ | |
18 --configuration "Debug" --target_platform "" --revision 0 \ | |
19 --make_flags "" --test_args "" --gm_args "" \ | |
20 --bench_args "" --perf_output_basedir "" \ | |
21 --builder_name Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release \ | |
22 --got_revision 0 --is_try False --dest_gsbase gs://rmistry | |
23 | |
24 """ | |
25 | |
26 import posixpath | |
27 import sys | |
28 | |
29 from build_step import BuildStep | |
30 from utils import old_gs_utils as gs_utils | |
31 from utils import sync_bucket_subdir | |
32 | |
33 | |
34 class CheckGoogleStorageTimestamps(BuildStep): | |
35 | |
36 def _Run(self): | |
37 dest_gsbase = (self._args.get('dest_gsbase') or | |
38 sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE) | |
39 | |
40 # gm-actual directories are of the form: | |
41 # gs://chromium-skia-gm/playback/gm-actual/platform/builder-name/platform/* | |
42 # We first get all directories under the first platform dir then get all | |
43 # directories under the builder-name and finally get the timestamp files | |
44 # from the final platform dir. | |
45 | |
46 # Get a list of all platform_dirs. | |
47 platform_dirs = gs_utils.list_storage_directory( | |
48 dest_gsbase=dest_gsbase, | |
49 subdir=posixpath.join( | |
50 self._storage_playback_dirs.PlaybackRootDir(), 'gm-actual')) | |
51 | |
52 # Get a list of all platform_builder_dirs. | |
53 platform_builder_dirs = [] | |
54 for platform_dir in platform_dirs: | |
55 platform_builder_dirs.extend(gs_utils.list_storage_directory( | |
56 dest_gsbase=platform_dir, | |
57 subdir='')) | |
58 # TODO(rmistry): Ignoring Trybot builders for now. Enable them when they | |
59 # are more stable. When I ran this script locally they were the only | |
60 # builders which had differing started and completed timestamps. | |
61 platform_builder_dirs = filter( | |
62 lambda x: not x.endswith(posixpath.join('_Trybot', '')), | |
63 platform_builder_dirs) | |
64 | |
65 # Get the final list of all platform_builder_platform_dirs. | |
66 platform_builder_platform_dirs = [] | |
67 for platform_and_builder_dir in platform_builder_dirs: | |
68 platform_builder_platform_dirs.extend(gs_utils.list_storage_directory( | |
69 dest_gsbase=platform_and_builder_dir, | |
70 subdir='')) | |
71 | |
72 # Check TIMESTAMP_LAST_UPLOAD_STARTED and TIMESTAMP_LAST_UPLOAD_COMPLETED in | |
73 # each platform_builder_platform directory. | |
74 failed_gm_actual_dirs = [] | |
75 for timestamp_dir in platform_builder_platform_dirs: | |
76 gm_actual_started_timestamp = gs_utils.read_timestamp_file( | |
77 timestamp_file_name=gs_utils.TIMESTAMP_STARTED_FILENAME, | |
78 gs_base=timestamp_dir, | |
79 gs_relative_dir='') | |
80 gm_actual_completed_timestamp = gs_utils.read_timestamp_file( | |
81 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | |
82 gs_base=timestamp_dir, | |
83 gs_relative_dir='') | |
84 if gm_actual_started_timestamp != gm_actual_completed_timestamp: | |
85 failed_gm_actual_dirs.append(timestamp_dir) | |
86 | |
87 if failed_gm_actual_dirs: | |
88 exception_txt = ( | |
89 '\n\nThese are the gm-actual directories with timestamps that do not ' | |
90 'match: %s\n' | |
91 'This indicates one of two things (can be determined by examining ' | |
92 'the directory):\n' | |
93 '* The builder is currently running and is in the process of ' | |
94 'updating its gm-actual directory. In this case we do not need to do ' | |
95 'anything.\n' | |
96 '* The builder\'s gm-actual directory is in an inconsistent state ' | |
97 'and needs to be manually fixed by deleting its ' | |
98 'TIMESTAMP_LAST_UPLOAD_COMPLETED directory.\n\n' | |
99 % failed_gm_actual_dirs) | |
100 exception_txt += ('Investigate the suspect directories with the following' | |
101 ' command(s):\n') | |
102 for failed_gm_actual_dir in failed_gm_actual_dirs: | |
103 exception_txt += 'gsutil ls -l %sTIMESTAMP*\n' % failed_gm_actual_dir | |
104 exception_txt += '\n' | |
105 raise Exception(exception_txt) | |
106 | |
107 | |
108 if '__main__' == __name__: | |
109 sys.exit(BuildStep.RunBuildStep(CheckGoogleStorageTimestamps)) | |
OLD | NEW |