OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # | 2 # |
3 # Copyright 2017 The Chromium Authors. All rights reserved. | 3 # Copyright 2017 The Chromium Authors. All rights reserved. |
4 # Use of this source code is governed by a BSD-style license that can be | 4 # Use of this source code is governed by a BSD-style license that can be |
5 # found in the LICENSE file. | 5 # found in the LICENSE file. |
6 | 6 |
7 import argparse | 7 import argparse |
| 8 import json |
8 import logging | 9 import logging |
9 import os | 10 import os |
| 11 import shutil |
10 import sys | 12 import sys |
11 import tempfile | 13 import tempfile |
12 | 14 |
13 from webkitpy.common.system.log_utils import configure_logging | 15 from webkitpy.common.system.log_utils import configure_logging |
14 from webkitpy.layout_tests import merge_results | 16 from webkitpy.layout_tests import merge_results |
15 | 17 |
16 # ------------------------------------------------------------------------ | 18 # ------------------------------------------------------------------------ |
17 | 19 |
18 | 20 |
19 def main(argv): | 21 def main(argv): |
(...skipping 22 matching lines...) Expand all Loading... |
42 parser.add_argument( | 44 parser.add_argument( |
43 '--results-json-allow-unknown-if-matching', | 45 '--results-json-allow-unknown-if-matching', |
44 action='store_true', default=False, | 46 action='store_true', default=False, |
45 help='Allow unknown values in the result.json file as long as the ' | 47 help='Allow unknown values in the result.json file as long as the ' |
46 'value match on all shards.') | 48 'value match on all shards.') |
47 | 49 |
48 parser.add_argument( | 50 parser.add_argument( |
49 '--output-directory', | 51 '--output-directory', |
50 help='Directory to create the merged results in.') | 52 help='Directory to create the merged results in.') |
51 parser.add_argument( | 53 parser.add_argument( |
| 54 '--allow-existing-output-directory', |
| 55 action='store_true', default=False, |
| 56 help='Allow merging results into a directory which already exists.') |
| 57 parser.add_argument( |
| 58 '--remove-existing-output-directory', |
| 59 action='store_true', default=False, |
| 60 help='Remove merging results into a directory which already exists.') |
| 61 parser.add_argument( |
52 '--input-directories', nargs='+', | 62 '--input-directories', nargs='+', |
53 help='Directories to merge the results from.') | 63 help='Directories to merge the results from.') |
54 | 64 |
55 # Swarming Isolated Merge Script API | 65 # Swarming Isolated Merge Script API |
56 # script.py --build-properties /s/build.json --output-json /tmp/output.json
shard0/output.json shard1/output.json | 66 # script.py --build-properties /s/build.json --output-json /tmp/output.json
shard0/output.json shard1/output.json |
57 parser.add_argument( | 67 parser.add_argument( |
58 '-o', '--output-json', | 68 '-o', '--output-json', |
59 help='(Swarming Isolated Merge Script API) Output JSON file to create.') | 69 help='(Swarming Isolated Merge Script API) Output JSON file to create.') |
60 parser.add_argument( | 70 parser.add_argument( |
61 '--build-properties', | 71 '--build-properties', |
62 help='(Swarming Isolated Merge Script API) Build property JSON file prov
ided by recipes.') | 72 help='(Swarming Isolated Merge Script API) Build property JSON file prov
ided by recipes.') |
| 73 parser.add_argument( |
| 74 '--results-json-override-with-build-property', |
| 75 nargs=2, metavar=('RESULT_JSON_KEY', 'BUILD_PROPERTY_KEY'), default=[], |
| 76 action='append', |
| 77 help='Override the value of a value in the result style JSON file ' |
| 78 '(--result-jsons-override-value layout_test_dirs /tmp/output).') |
63 | 79 |
64 # Script to run after merging the directories together. Normally used with a
rchive_layout_test_results.py | 80 # Script to run after merging the directories together. Normally used with a
rchive_layout_test_results.py |
65 # scripts/slave/chromium/archive_layout_test_results.py \ | 81 # scripts/slave/chromium/archive_layout_test_results.py \ |
66 # --results-dir /b/rr/tmpIcChUS/w/layout-test-results \ | 82 # --results-dir /b/rr/tmpIcChUS/w/layout-test-results \ |
67 # --build-dir /b/rr/tmpIcChUS/w/src/out \ | 83 # --build-dir /b/rr/tmpIcChUS/w/src/out \ |
68 # --build-number 3665 \ | 84 # --build-number 3665 \ |
69 # --builder-name 'WebKit Linux - RandomOrder' \ | 85 # --builder-name 'WebKit Linux - RandomOrder' \ |
70 # --gs-bucket gs://chromium-layout-test-archives \ | 86 # --gs-bucket gs://chromium-layout-test-archives \ |
71 # --staging-dir /b/c/chrome_staging \ | 87 # --staging-dir /b/c/chrome_staging \ |
72 # --slave-utils-gsutil-py-path /b/rr/tmpIcChUS/rw/scripts/slave/.recipe_
deps/depot_tools/gsutil.py | 88 # --slave-utils-gsutil-py-path /b/rr/tmpIcChUS/rw/scripts/slave/.recipe_
deps/depot_tools/gsutil.py |
73 # in dir /b/rr/tmpIcChUS/w | 89 # in dir /b/rr/tmpIcChUS/w |
74 parser.add_argument( | 90 parser.add_argument( |
75 '--post-merge-script', | 91 '--post-merge-script', |
76 nargs='*', | 92 nargs='*', |
77 help='Script to call after the results have been merged.') | 93 help='Script to call after the results have been merged.') |
78 | 94 |
79 # The position arguments depend on if we are using the isolated merge | 95 # The position arguments depend on if we are using the isolated merge |
80 # script API mode or not. | 96 # script API mode or not. |
81 parser.add_argument( | 97 parser.add_argument( |
82 'positional', nargs='*', | 98 'positional', nargs='*', |
83 help='output.json from shards.') | 99 help='output.json from shards.') |
84 | 100 |
85 args = parser.parse_args(argv) | 101 args = parser.parse_args(argv) |
86 if args.verbose: | 102 if args.verbose: |
87 logging_level = logging.DEBUG | 103 logging_level = logging.DEBUG |
88 else: | 104 else: |
89 logging_level = logging.INFO | 105 logging_level = logging.INFO |
90 configure_logging(logging_level=logging_level) | 106 configure_logging(logging_level=logging_level) |
91 | 107 |
| 108 results_json_value_overrides = {} |
| 109 |
92 # Map the isolate arguments back to our output / input arguments. | 110 # Map the isolate arguments back to our output / input arguments. |
93 if args.output_json: | 111 if args.output_json: |
| 112 logging.info('Running with isolated arguments') |
94 assert args.positional | 113 assert args.positional |
95 | 114 |
| 115 if args.results_json_override_with_build_property: |
| 116 if not args.build_properties: |
| 117 parser.error( |
| 118 '--results-json-override-with-build-property given' |
| 119 ' but --build-properties was not.') |
| 120 build_properties = json.loads(args.build_properties) |
| 121 for result_key, build_prop_key in args.results_json_override_with_bu
ild_property: |
| 122 results_json_value_overrides[result_key] = build_properties[buil
d_prop_key] |
| 123 |
96 assert not args.output_directory | 124 assert not args.output_directory |
97 args.output_directory = os.path.dirname(args.output_json) | 125 args.output_directory = os.getcwd() |
| 126 args.remove_existing_output_directory = True |
98 | 127 |
99 assert not args.input_directories | 128 assert not args.input_directories |
100 args.input_directories = [os.path.dirname(f) for f in args.positional] | 129 args.input_directories = [os.path.dirname(f) for f in args.positional] |
101 args.positional = [] | 130 args.positional = [] |
102 | 131 |
103 # Allow skipping the --input-directories bit, for example, | 132 # Allow skipping the --input-directories bit, for example, |
104 # merge-layout-test-results -o outputdir shard0 shard1 shard2 | 133 # merge-layout-test-results -o outputdir shard0 shard1 shard2 |
105 if args.positional and not args.input_directories: | 134 if args.positional and not args.input_directories: |
106 args.input_directories = args.positional | 135 args.input_directories = args.positional |
107 | 136 |
108 if not args.output_directory: | 137 if not args.output_directory: |
109 args.output_directory = tempfile.mkdtemp(suffix='webkit_layout_test_resu
lts.') | 138 args.output_directory = tempfile.mkdtemp(suffix='webkit_layout_test_resu
lts.') |
110 | 139 |
111 assert args.output_directory | 140 assert args.output_directory |
112 assert args.input_directories | 141 assert args.input_directories |
113 | 142 |
114 results_json_value_overrides = {} | |
115 for k, v in args.results_json_override_value: | 143 for k, v in args.results_json_override_value: |
116 assert k not in results_json_value_overrides | 144 assert k not in results_json_value_overrides |
117 try: | 145 try: |
118 results_json_value_overrides[k] = eval(v) | 146 results_json_value_overrides[k] = eval(v) |
119 except NameError: | 147 except NameError: |
120 results_json_value_overrides[k] = v | 148 results_json_value_overrides[k] = v |
| 149 logging.debug('results_json_value_overrides: %r', results_json_value_overrid
es) |
121 | 150 |
122 merger = merge_results.LayoutTestDirMerger( | 151 merger = merge_results.LayoutTestDirMerger( |
123 results_json_value_overrides=results_json_value_overrides, | 152 results_json_value_overrides=results_json_value_overrides, |
124 results_json_allow_unknown_if_matching=args.results_json_allow_unknown_i
f_matching) | 153 results_json_allow_unknown_if_matching=args.results_json_allow_unknown_i
f_matching) |
| 154 if os.path.exists(args.output_directory): |
| 155 logging.warning('Output directory exists %r', args.output_directory) |
| 156 if args.remove_existing_output_directory: |
| 157 shutil.rmtree(args.output_directory) |
| 158 elif not args.allow_existing_output_directory: |
| 159 raise IOError( |
| 160 ('Output directory %s exists!\n' |
| 161 'Use --allow-existing-output-directory to continue') % args.out
put_directory) |
| 162 |
125 merger.merge(args.output_directory, args.input_directories) | 163 merger.merge(args.output_directory, args.input_directories) |
126 | 164 |
| 165 merged_output_json = os.path.join(args.output_directory, 'output.json') |
| 166 if os.path.exists(merged_output_json) and args.output_json: |
| 167 logging.debug( |
| 168 'Copying output.json from %s to %s', merged_output_json, args.output
_json) |
| 169 shutil.copyfile(merged_output_json, args.output_json) |
| 170 |
127 if args.post_merge_script: | 171 if args.post_merge_script: |
| 172 logging.debug('Changing directory to %s', args.output_directory) |
128 os.chdir(args.output_directory) | 173 os.chdir(args.output_directory) |
129 | 174 |
130 post_script = list(args.post_merge_script) | 175 post_script = list(args.post_merge_script) |
131 post_script.append('--result-dir', args.output_directory) | 176 post_script.append('--result-dir', args.output_directory) |
132 | 177 |
| 178 logging.info('Running post merge script %r', post_script) |
133 os.execlp(post_script) | 179 os.execlp(post_script) |
134 | 180 |
135 main(sys.argv[1:]) | 181 main(sys.argv[1:]) |
OLD | NEW |