Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(836)

Side by Side Diff: third_party/WebKit/Tools/Scripts/merge-layout-test-results

Issue 2896423004: webkitpy: Remove contents of directory. (Closed)
Patch Set: Fixing tests. Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | third_party/WebKit/Tools/Scripts/webkitpy/common/system/filesystem.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 # 2 #
3 # Copyright 2017 The Chromium Authors. All rights reserved. 3 # Copyright 2017 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be 4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file. 5 # found in the LICENSE file.
6 6
7 import argparse 7 import argparse
8 import json 8 import json
9 import logging 9 import logging
10 import os 10 import os
11 import stat
11 import shutil 12 import shutil
12 import sys 13 import sys
13 import tempfile 14 import tempfile
14 import time 15 import time
15 16
17 from webkitpy.common.system.filesystem import FileSystem
16 from webkitpy.common.system.log_utils import configure_logging 18 from webkitpy.common.system.log_utils import configure_logging
17 from webkitpy.layout_tests import merge_results 19 from webkitpy.layout_tests import merge_results
18 20
19 # ------------------------------------------------------------------------ 21 # ------------------------------------------------------------------------
22 def ensure_empty_dir(fs, directory, allow_existing, remove_existing):
23 """Ensure an empty directory exists.
20 24
21 def rmtree(dirname): 25 Args:
22 # Attempt to remove a directory tree. We try multiple times as on Windows a 26 allow_existing (bool): Allow the empty directory to already exist.
23 # process which is currently closing could still have a file open in the 27 remove_existing (bool): Remove the contents if the directory
24 # directory. 28 already exists.
25 logging.info('Removing %s', dirname) 29 """
26 errors = [] 30 if not fs.exists(directory):
27 def onerror(func, path, exc_info): 31 fs.maybe_make_directory(directory)
28 errors.append(path)
29 logging.exception('Failed at %s %s: %r', func, path, exc_info)
30
31 attempts = 0
32 while attempts < 5:
33 del errors[:]
34 shutil.rmtree(dirname, onerror=onerror)
35 if not errors:
36 break
37 attempts += 1
38 time.sleep(1)
39
40 # Check the path is gone.
41 if not os.path.exists(dirname):
42 return 32 return
43 33
44 logging.warning('Unable to remove %s', dirname) 34 logging.warning('Output directory exists %r', directory)
45 for dirpath, dirnames, filenames in os.walk(dirname, onerror=onerror, topdow n=False): 35 if not allow_existing:
46 for fname in filenames: 36 raise IOError(
47 logging.warning('File %s still in output dir.', os.path.join(dirpath , fname)) 37 ('Output directory %s exists!\n'
48 for dname in dirnames: 38 'Use --allow-existing-output-directory to continue') % directory)
49 logging.warning('Dir %s still in output dir.', os.path.join(dirpath, dname)) 39
40 if remove_existing and not fs.remove_contents(directory):
41 raise IOError(
42 ('Unable to remove output directory %s contents!\n'
43 'See log output for errors.') % directory)
50 44
51 45
52 def main(argv): 46 def main(argv):
53 47
54 parser = argparse.ArgumentParser() 48 parser = argparse.ArgumentParser()
55 parser.description = """\ 49 parser.description = """\
56 Merges sharded layout test results into a single output directory. 50 Merges sharded layout test results into a single output directory.
57 """ 51 """
58 parser.epilog = """\ 52 parser.epilog = """\
59 53
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
149 143
150 if args.results_json_override_with_build_property: 144 if args.results_json_override_with_build_property:
151 if not args.build_properties: 145 if not args.build_properties:
152 parser.error( 146 parser.error(
153 '--results-json-override-with-build-property given' 147 '--results-json-override-with-build-property given'
154 ' but --build-properties was not.') 148 ' but --build-properties was not.')
155 build_properties = json.loads(args.build_properties) 149 build_properties = json.loads(args.build_properties)
156 for result_key, build_prop_key in args.results_json_override_with_bu ild_property: 150 for result_key, build_prop_key in args.results_json_override_with_bu ild_property:
157 results_json_value_overrides[result_key] = build_properties[buil d_prop_key] 151 results_json_value_overrides[result_key] = build_properties[buil d_prop_key]
158 152
159 assert not args.output_directory 153 if not args.output_directory:
160 args.output_directory = os.getcwd() 154 args.output_directory = os.getcwd()
161 args.remove_existing_output_directory = True 155 args.allow_existing_output_directory = True
156 args.remove_existing_output_directory = True
162 157
163 assert not args.input_directories 158 assert not args.input_directories
164 args.input_directories = [os.path.dirname(f) for f in args.positional] 159 args.input_directories = [os.path.dirname(f) for f in args.positional]
165 args.positional = [] 160 args.positional = []
166 161
167 # Allow skipping the --input-directories bit, for example, 162 # Allow skipping the --input-directories bit, for example,
168 # merge-layout-test-results -o outputdir shard0 shard1 shard2 163 # merge-layout-test-results -o outputdir shard0 shard1 shard2
169 if args.positional and not args.input_directories: 164 if args.positional and not args.input_directories:
170 args.input_directories = args.positional 165 args.input_directories = args.positional
171 166
172 if not args.output_directory: 167 if not args.output_directory:
173 args.output_directory = tempfile.mkdtemp(suffix='webkit_layout_test_resu lts.') 168 args.output_directory = tempfile.mkdtemp(suffix='webkit_layout_test_resu lts.')
174 169
175 assert args.output_directory 170 assert args.output_directory
176 assert args.input_directories 171 assert args.input_directories
177 172
178 for k, v in args.results_json_override_value: 173 for k, v in args.results_json_override_value:
179 assert k not in results_json_value_overrides 174 assert k not in results_json_value_overrides
180 try: 175 try:
181 results_json_value_overrides[k] = eval(v) 176 results_json_value_overrides[k] = eval(v)
182 except NameError: 177 except NameError:
183 results_json_value_overrides[k] = v 178 results_json_value_overrides[k] = v
184 logging.debug('results_json_value_overrides: %r', results_json_value_overrid es) 179 logging.debug('results_json_value_overrides: %r', results_json_value_overrid es)
185 180
186 merger = merge_results.LayoutTestDirMerger( 181 merger = merge_results.LayoutTestDirMerger(
187 results_json_value_overrides=results_json_value_overrides, 182 results_json_value_overrides=results_json_value_overrides,
188 results_json_allow_unknown_if_matching=args.results_json_allow_unknown_i f_matching) 183 results_json_allow_unknown_if_matching=args.results_json_allow_unknown_i f_matching)
189 if os.path.exists(args.output_directory): 184
190 logging.warning('Output directory exists %r', args.output_directory) 185 ensure_empty_dir(
191 if args.remove_existing_output_directory: 186 FileSystem(),
192 rmtree(args.output_directory) 187 args.output_directory,
193 elif not args.allow_existing_output_directory: 188 allow_existing=args.allow_existing_output_directory,
194 raise IOError( 189 remove_existing=args.remove_existing_output_directory)
195 ('Output directory %s exists!\n'
196 'Use --allow-existing-output-directory to continue') % args.out put_directory)
197 190
198 merger.merge(args.output_directory, args.input_directories) 191 merger.merge(args.output_directory, args.input_directories)
199 192
200 merged_output_json = os.path.join(args.output_directory, 'output.json') 193 merged_output_json = os.path.join(args.output_directory, 'output.json')
201 if os.path.exists(merged_output_json) and args.output_json: 194 if os.path.exists(merged_output_json) and args.output_json:
202 logging.debug( 195 logging.debug(
203 'Copying output.json from %s to %s', merged_output_json, args.output _json) 196 'Copying output.json from %s to %s', merged_output_json, args.output _json)
204 shutil.copyfile(merged_output_json, args.output_json) 197 shutil.copyfile(merged_output_json, args.output_json)
205 198
206 if args.post_merge_script: 199 if args.post_merge_script:
207 logging.debug('Changing directory to %s', args.output_directory) 200 logging.debug('Changing directory to %s', args.output_directory)
208 os.chdir(args.output_directory) 201 os.chdir(args.output_directory)
209 202
210 post_script = list(args.post_merge_script) 203 post_script = list(args.post_merge_script)
211 post_script.append('--result-dir', args.output_directory) 204 post_script.append('--result-dir', args.output_directory)
212 205
213 logging.info('Running post merge script %r', post_script) 206 logging.info('Running post merge script %r', post_script)
214 os.execlp(post_script) 207 os.execlp(post_script)
215 208
216 main(sys.argv[1:]) 209 main(sys.argv[1:])
OLDNEW
« no previous file with comments | « no previous file | third_party/WebKit/Tools/Scripts/webkitpy/common/system/filesystem.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698