| OLD | NEW |
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Checks that the timestamps of gm-actual directories on Google Storage match. | |
| 7 | |
| 8 Checks if the TIMESTAMP_LAST_UPLOAD_STARTED and TIMESTAMP_LAST_UPLOAD_COMPLETED | |
| 9 files of all gm-actual directories on Google Storage match. This module can be | |
| 10 run from the command-line like this: | |
| 11 | |
| 12 cd buildbot/third_party/chromium_buildbot/slave/\ | |
| 13 Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release/build/trunk | |
| 14 | |
| 15 PYTHONPATH=../../../../scripts:\ | |
| 16 ../../../../site_config \ | |
| 17 python ../../../../../../slave/skia_slave_scripts/check_gs_timestamps.py \ | |
| 18 --configuration "Debug" --target_platform "" --revision 0 \ | |
| 19 --make_flags "" --test_args "" --gm_args "" \ | |
| 20 --bench_args "" --perf_output_basedir "" \ | |
| 21 --builder_name Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release \ | |
| 22 --got_revision 0 --is_try False --dest_gsbase gs://rmistry | |
| 23 | |
| 24 """ | |
| 25 | |
| 26 import posixpath | |
| 27 import sys | |
| 28 | |
| 29 from build_step import BuildStep | |
| 30 from utils import old_gs_utils as gs_utils | |
| 31 from utils import sync_bucket_subdir | |
| 32 | |
| 33 | |
| 34 class CheckGoogleStorageTimestamps(BuildStep): | |
| 35 | |
| 36 def _Run(self): | |
| 37 dest_gsbase = (self._args.get('dest_gsbase') or | |
| 38 sync_bucket_subdir.DEFAULT_PERFDATA_GS_BASE) | |
| 39 | |
| 40 # gm-actual directories are of the form: | |
| 41 # gs://chromium-skia-gm/playback/gm-actual/platform/builder-name/platform/* | |
| 42 # We first get all directories under the first platform dir then get all | |
| 43 # directories under the builder-name and finally get the timestamp files | |
| 44 # from the final platform dir. | |
| 45 | |
| 46 # Get a list of all platform_dirs. | |
| 47 platform_dirs = gs_utils.list_storage_directory( | |
| 48 dest_gsbase=dest_gsbase, | |
| 49 subdir=posixpath.join( | |
| 50 self._storage_playback_dirs.PlaybackRootDir(), 'gm-actual')) | |
| 51 | |
| 52 # Get a list of all platform_builder_dirs. | |
| 53 platform_builder_dirs = [] | |
| 54 for platform_dir in platform_dirs: | |
| 55 platform_builder_dirs.extend(gs_utils.list_storage_directory( | |
| 56 dest_gsbase=platform_dir, | |
| 57 subdir='')) | |
| 58 # TODO(rmistry): Ignoring Trybot builders for now. Enable them when they | |
| 59 # are more stable. When I ran this script locally they were the only | |
| 60 # builders which had differing started and completed timestamps. | |
| 61 platform_builder_dirs = filter( | |
| 62 lambda x: not x.endswith(posixpath.join('_Trybot', '')), | |
| 63 platform_builder_dirs) | |
| 64 | |
| 65 # Get the final list of all platform_builder_platform_dirs. | |
| 66 platform_builder_platform_dirs = [] | |
| 67 for platform_and_builder_dir in platform_builder_dirs: | |
| 68 platform_builder_platform_dirs.extend(gs_utils.list_storage_directory( | |
| 69 dest_gsbase=platform_and_builder_dir, | |
| 70 subdir='')) | |
| 71 | |
| 72 # Check TIMESTAMP_LAST_UPLOAD_STARTED and TIMESTAMP_LAST_UPLOAD_COMPLETED in | |
| 73 # each platform_builder_platform directory. | |
| 74 failed_gm_actual_dirs = [] | |
| 75 for timestamp_dir in platform_builder_platform_dirs: | |
| 76 gm_actual_started_timestamp = gs_utils.read_timestamp_file( | |
| 77 timestamp_file_name=gs_utils.TIMESTAMP_STARTED_FILENAME, | |
| 78 gs_base=timestamp_dir, | |
| 79 gs_relative_dir='') | |
| 80 gm_actual_completed_timestamp = gs_utils.read_timestamp_file( | |
| 81 timestamp_file_name=gs_utils.TIMESTAMP_COMPLETED_FILENAME, | |
| 82 gs_base=timestamp_dir, | |
| 83 gs_relative_dir='') | |
| 84 if gm_actual_started_timestamp != gm_actual_completed_timestamp: | |
| 85 failed_gm_actual_dirs.append(timestamp_dir) | |
| 86 | |
| 87 if failed_gm_actual_dirs: | |
| 88 exception_txt = ( | |
| 89 '\n\nThese are the gm-actual directories with timestamps that do not ' | |
| 90 'match: %s\n' | |
| 91 'This indicates one of two things (can be determined by examining ' | |
| 92 'the directory):\n' | |
| 93 '* The builder is currently running and is in the process of ' | |
| 94 'updating its gm-actual directory. In this case we do not need to do ' | |
| 95 'anything.\n' | |
| 96 '* The builder\'s gm-actual directory is in an inconsistent state ' | |
| 97 'and needs to be manually fixed by deleting its ' | |
| 98 'TIMESTAMP_LAST_UPLOAD_COMPLETED directory.\n\n' | |
| 99 % failed_gm_actual_dirs) | |
| 100 exception_txt += ('Investigate the suspect directories with the following' | |
| 101 ' command(s):\n') | |
| 102 for failed_gm_actual_dir in failed_gm_actual_dirs: | |
| 103 exception_txt += 'gsutil ls -l %sTIMESTAMP*\n' % failed_gm_actual_dir | |
| 104 exception_txt += '\n' | |
| 105 raise Exception(exception_txt) | |
| 106 | |
| 107 | |
| 108 if '__main__' == __name__: | |
| 109 sys.exit(BuildStep.RunBuildStep(CheckGoogleStorageTimestamps)) | |
| OLD | NEW |