| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 # | 2 # |
| 3 # Copyright 2017 The Chromium Authors. All rights reserved. | 3 # Copyright 2017 The Chromium Authors. All rights reserved. |
| 4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
| 5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
| 6 | 6 |
| 7 import argparse | 7 import argparse |
| 8 import json | 8 import json |
| 9 import logging | 9 import logging |
| 10 import os | 10 import os |
| 11 import stat |
| 11 import shutil | 12 import shutil |
| 12 import sys | 13 import sys |
| 13 import tempfile | 14 import tempfile |
| 14 import time | 15 import time |
| 15 | 16 |
| 17 from webkitpy.common.system.filesystem import FileSystem |
| 16 from webkitpy.common.system.log_utils import configure_logging | 18 from webkitpy.common.system.log_utils import configure_logging |
| 17 from webkitpy.layout_tests import merge_results | 19 from webkitpy.layout_tests import merge_results |
| 18 | 20 |
| 19 # ------------------------------------------------------------------------ | 21 # ------------------------------------------------------------------------ |
| 22 def ensure_empty_dir(fs, directory, allow_existing, remove_existing): |
| 23 """Ensure an empty directory exists. |
| 20 | 24 |
| 21 def rmtree(dirname): | 25 Args: |
| 22 # Attempt to remove a directory tree. We try multiple times as on Windows a | 26 allow_existing (bool): Allow the empty directory to already exist. |
| 23 # process which is currently closing could still have a file open in the | 27 remove_existing (bool): Remove the contents if the directory |
| 24 # directory. | 28 already exists. |
| 25 logging.info('Removing %s', dirname) | 29 """ |
| 26 errors = [] | 30 if not fs.exists(directory): |
| 27 def onerror(func, path, exc_info): | 31 fs.maybe_make_directory(directory) |
| 28 errors.append(path) | |
| 29 logging.exception('Failed at %s %s: %r', func, path, exc_info) | |
| 30 | |
| 31 attempts = 0 | |
| 32 while attempts < 5: | |
| 33 del errors[:] | |
| 34 shutil.rmtree(dirname, onerror=onerror) | |
| 35 if not errors: | |
| 36 break | |
| 37 attempts += 1 | |
| 38 time.sleep(1) | |
| 39 | |
| 40 # Check the path is gone. | |
| 41 if not os.path.exists(dirname): | |
| 42 return | 32 return |
| 43 | 33 |
| 44 logging.warning('Unable to remove %s', dirname) | 34 logging.warning('Output directory exists %r', directory) |
| 45 for dirpath, dirnames, filenames in os.walk(dirname, onerror=onerror, topdow
n=False): | 35 if not allow_existing: |
| 46 for fname in filenames: | 36 raise IOError( |
| 47 logging.warning('File %s still in output dir.', os.path.join(dirpath
, fname)) | 37 ('Output directory %s exists!\n' |
| 48 for dname in dirnames: | 38 'Use --allow-existing-output-directory to continue') % directory) |
| 49 logging.warning('Dir %s still in output dir.', os.path.join(dirpath,
dname)) | 39 |
| 40 if remove_existing and not fs.remove_contents(directory): |
| 41 raise IOError( |
| 42 ('Unable to remove output directory %s contents!\n' |
| 43 'See log output for errors.') % directory) |
| 50 | 44 |
| 51 | 45 |
| 52 def main(argv): | 46 def main(argv): |
| 53 | 47 |
| 54 parser = argparse.ArgumentParser() | 48 parser = argparse.ArgumentParser() |
| 55 parser.description = """\ | 49 parser.description = """\ |
| 56 Merges sharded layout test results into a single output directory. | 50 Merges sharded layout test results into a single output directory. |
| 57 """ | 51 """ |
| 58 parser.epilog = """\ | 52 parser.epilog = """\ |
| 59 | 53 |
| (...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 149 | 143 |
| 150 if args.results_json_override_with_build_property: | 144 if args.results_json_override_with_build_property: |
| 151 if not args.build_properties: | 145 if not args.build_properties: |
| 152 parser.error( | 146 parser.error( |
| 153 '--results-json-override-with-build-property given' | 147 '--results-json-override-with-build-property given' |
| 154 ' but --build-properties was not.') | 148 ' but --build-properties was not.') |
| 155 build_properties = json.loads(args.build_properties) | 149 build_properties = json.loads(args.build_properties) |
| 156 for result_key, build_prop_key in args.results_json_override_with_bu
ild_property: | 150 for result_key, build_prop_key in args.results_json_override_with_bu
ild_property: |
| 157 results_json_value_overrides[result_key] = build_properties[buil
d_prop_key] | 151 results_json_value_overrides[result_key] = build_properties[buil
d_prop_key] |
| 158 | 152 |
| 159 assert not args.output_directory | 153 if not args.output_directory: |
| 160 args.output_directory = os.getcwd() | 154 args.output_directory = os.getcwd() |
| 161 args.remove_existing_output_directory = True | 155 args.allow_existing_output_directory = True |
| 156 args.remove_existing_output_directory = True |
| 162 | 157 |
| 163 assert not args.input_directories | 158 assert not args.input_directories |
| 164 args.input_directories = [os.path.dirname(f) for f in args.positional] | 159 args.input_directories = [os.path.dirname(f) for f in args.positional] |
| 165 args.positional = [] | 160 args.positional = [] |
| 166 | 161 |
| 167 # Allow skipping the --input-directories bit, for example, | 162 # Allow skipping the --input-directories bit, for example, |
| 168 # merge-layout-test-results -o outputdir shard0 shard1 shard2 | 163 # merge-layout-test-results -o outputdir shard0 shard1 shard2 |
| 169 if args.positional and not args.input_directories: | 164 if args.positional and not args.input_directories: |
| 170 args.input_directories = args.positional | 165 args.input_directories = args.positional |
| 171 | 166 |
| 172 if not args.output_directory: | 167 if not args.output_directory: |
| 173 args.output_directory = tempfile.mkdtemp(suffix='webkit_layout_test_resu
lts.') | 168 args.output_directory = tempfile.mkdtemp(suffix='webkit_layout_test_resu
lts.') |
| 174 | 169 |
| 175 assert args.output_directory | 170 assert args.output_directory |
| 176 assert args.input_directories | 171 assert args.input_directories |
| 177 | 172 |
| 178 for k, v in args.results_json_override_value: | 173 for k, v in args.results_json_override_value: |
| 179 assert k not in results_json_value_overrides | 174 assert k not in results_json_value_overrides |
| 180 try: | 175 try: |
| 181 results_json_value_overrides[k] = eval(v) | 176 results_json_value_overrides[k] = eval(v) |
| 182 except NameError: | 177 except NameError: |
| 183 results_json_value_overrides[k] = v | 178 results_json_value_overrides[k] = v |
| 184 logging.debug('results_json_value_overrides: %r', results_json_value_overrid
es) | 179 logging.debug('results_json_value_overrides: %r', results_json_value_overrid
es) |
| 185 | 180 |
| 186 merger = merge_results.LayoutTestDirMerger( | 181 merger = merge_results.LayoutTestDirMerger( |
| 187 results_json_value_overrides=results_json_value_overrides, | 182 results_json_value_overrides=results_json_value_overrides, |
| 188 results_json_allow_unknown_if_matching=args.results_json_allow_unknown_i
f_matching) | 183 results_json_allow_unknown_if_matching=args.results_json_allow_unknown_i
f_matching) |
| 189 if os.path.exists(args.output_directory): | 184 |
| 190 logging.warning('Output directory exists %r', args.output_directory) | 185 ensure_empty_dir( |
| 191 if args.remove_existing_output_directory: | 186 FileSystem(), |
| 192 rmtree(args.output_directory) | 187 args.output_directory, |
| 193 elif not args.allow_existing_output_directory: | 188 allow_existing=args.allow_existing_output_directory, |
| 194 raise IOError( | 189 remove_existing=args.remove_existing_output_directory) |
| 195 ('Output directory %s exists!\n' | |
| 196 'Use --allow-existing-output-directory to continue') % args.out
put_directory) | |
| 197 | 190 |
| 198 merger.merge(args.output_directory, args.input_directories) | 191 merger.merge(args.output_directory, args.input_directories) |
| 199 | 192 |
| 200 merged_output_json = os.path.join(args.output_directory, 'output.json') | 193 merged_output_json = os.path.join(args.output_directory, 'output.json') |
| 201 if os.path.exists(merged_output_json) and args.output_json: | 194 if os.path.exists(merged_output_json) and args.output_json: |
| 202 logging.debug( | 195 logging.debug( |
| 203 'Copying output.json from %s to %s', merged_output_json, args.output
_json) | 196 'Copying output.json from %s to %s', merged_output_json, args.output
_json) |
| 204 shutil.copyfile(merged_output_json, args.output_json) | 197 shutil.copyfile(merged_output_json, args.output_json) |
| 205 | 198 |
| 206 if args.post_merge_script: | 199 if args.post_merge_script: |
| 207 logging.debug('Changing directory to %s', args.output_directory) | 200 logging.debug('Changing directory to %s', args.output_directory) |
| 208 os.chdir(args.output_directory) | 201 os.chdir(args.output_directory) |
| 209 | 202 |
| 210 post_script = list(args.post_merge_script) | 203 post_script = list(args.post_merge_script) |
| 211 post_script.append('--result-dir', args.output_directory) | 204 post_script.append('--result-dir', args.output_directory) |
| 212 | 205 |
| 213 logging.info('Running post merge script %r', post_script) | 206 logging.info('Running post merge script %r', post_script) |
| 214 os.execlp(post_script) | 207 os.execlp(post_script) |
| 215 | 208 |
| 216 main(sys.argv[1:]) | 209 main(sys.argv[1:]) |
| OLD | NEW |