OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 | |
3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
4 # Use of this source code is governed by a BSD-style license that can be | |
5 # found in the LICENSE file. | |
6 | |
7 | |
8 """ Look through skia-autogen, searching for all checksums which should have | |
9 corresponding files in Google Storage, and verify that those files exist. """ | |
10 | |
11 | |
12 import json | |
13 import posixpath | |
14 import re | |
15 import subprocess | |
16 import sys | |
17 | |
18 | |
19 # TODO(borenet): Replace some/all of these with constants from gm/gm_json.py | |
20 AUTOGEN_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual' | |
21 GS_URL = 'gs://chromium-skia-gm/gm' | |
22 TEST_NAME_PATTERN = re.compile('(\S+)_(\S+).png') | |
23 | |
24 | |
25 def FileNameToGSURL(filename, hash_type, hash_value): | |
26 """ Convert a file name given in a checksum file to the URL of the | |
27 corresponding image file in Google Storage. | |
28 | |
29 filename: string; the file name to convert. Takes the form specified by | |
30 TEST_NAME_PATTERN. | |
31 hash_type: string; the type of the checksum. | |
32 hash_value: string; the checksum itself. | |
33 """ | |
34 test_name = TEST_NAME_PATTERN.match(filename).group(1) | |
35 if not test_name: | |
36 raise Exception('Invalid test name for file: %s' % filename) | |
37 return '%s/%s/%s/%s.png' % (GS_URL, hash_type, test_name, hash_value) | |
38 | |
39 | |
40 def FindURLSInJSON(json_file, gs_urls): | |
41 """ Extract Google Storage URLs from a JSON file in svn, adding them to the | |
42 gs_urls dictionary. | |
43 | |
44 json_file: string; URL of the JSON file. | |
45 gs_urls: dict; stores Google Storage URLs as keys and lists of the JSON files | |
46 which reference them. | |
47 | |
48 Example gs_urls: | |
49 { 'gs://chromium-skia-gm/gm/sometest/12345.png': [ | |
50 'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.6-MacMini4.1-
GeForce320M-x86-Debug/actual-results.json', | |
51 'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.8-MacMini4.1-
GeForce320M-x86-Debug/actual-results.json', | |
52 ] | |
53 } | |
54 """ | |
55 output = subprocess.check_output(['svn', 'cat', json_file]) | |
56 json_content = json.loads(output) | |
57 for dict_type in ['actual-results']: | |
58 for result_type in json_content[dict_type]: | |
59 if json_content[dict_type][result_type]: | |
60 for result in json_content[dict_type][result_type].keys(): | |
61 hash_type, hash_value = json_content[dict_type][result_type][result] | |
62 gs_url = FileNameToGSURL(result, hash_type, str(hash_value)) | |
63 if gs_urls.get(gs_url): | |
64 gs_urls[gs_url].append(json_file) | |
65 else: | |
66 gs_urls[gs_url] = [json_file] | |
67 | |
68 | |
69 def _FindJSONFiles(url, json_files): | |
70 """ Helper function for FindJsonFiles. Recursively explore the repository, | |
71 adding JSON files to a list. | |
72 | |
73 url: string; URL of the repository (or subdirectory thereof) to explore. | |
74 json_files: list to which JSON file urls will be added. | |
75 """ | |
76 proc = subprocess.Popen(['svn', 'ls', url], stdout=subprocess.PIPE, | |
77 stderr=subprocess.STDOUT) | |
78 if proc.wait() != 0: | |
79 raise Exception('Failed to list svn directory.') | |
80 output = proc.communicate()[0].splitlines() | |
81 subdirs = [] | |
82 for item in output: | |
83 if item.endswith(posixpath.sep): | |
84 subdirs.append(item) | |
85 elif item.endswith('.json'): | |
86 json_files.append(posixpath.join(url, item)) | |
87 else: | |
88 print 'Warning: ignoring %s' % posixpath.join(url, item) | |
89 for subdir in subdirs: | |
90 _FindJSONFiles(posixpath.join(url, subdir), json_files) | |
91 | |
92 | |
93 def FindJSONFiles(url): | |
94 """ Recursively explore the given repository and return a list of the JSON | |
95 files it contains. | |
96 | |
97 url: string; URL of the repository to explore. | |
98 """ | |
99 print 'Searching for JSON files in %s' % url | |
100 json_files = [] | |
101 _FindJSONFiles(url, json_files) | |
102 return json_files | |
103 | |
104 | |
105 def FindURLs(url): | |
106 """ Find Google Storage URLs inside of JSON files in the given repository. | |
107 Returns a dictionary whose keys are Google Storage URLs and values are lists | |
108 of the JSON files which reference them. | |
109 | |
110 url: string; URL of the repository to explore. | |
111 | |
112 Example output: | |
113 { 'gs://chromium-skia-gm/gm/sometest/12345.png': [ | |
114 'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.6-MacMini4.1-
GeForce320M-x86-Debug/actual-results.json', | |
115 'http://skia-autogen.googlecode.com/svn/gm-actual/Test-Mac10.8-MacMini4.1-
GeForce320M-x86-Debug/actual-results.json', | |
116 ] | |
117 } | |
118 """ | |
119 gs_urls = {} | |
120 for json_file in FindJSONFiles(url): | |
121 print 'Looking for checksums in %s' % json_file | |
122 FindURLSInJSON(json_file, gs_urls) | |
123 return gs_urls | |
124 | |
125 | |
126 def VerifyURL(url): | |
127 """ Verify that the given URL exists. | |
128 | |
129 url: string; the Google Storage URL of the image file in question. | |
130 """ | |
131 proc = subprocess.Popen(['gsutil', 'ls', url], stdout=subprocess.PIPE, | |
132 stderr=subprocess.STDOUT) | |
133 if proc.wait() != 0: | |
134 return False | |
135 return True | |
136 | |
137 | |
138 def VerifyURLs(urls): | |
139 """ Verify that each of the given URLs exists. Return a list of which URLs do | |
140 not exist. | |
141 | |
142 urls: dictionary; URLs of the image files in question. | |
143 """ | |
144 print 'Verifying that images exist for URLs...' | |
145 missing = [] | |
146 for url in urls.iterkeys(): | |
147 if not VerifyURL(url): | |
148 print 'Missing: %s, referenced by: \n %s' % (url, '\n '.join(urls[url])) | |
149 missing.append(url) | |
150 return missing | |
151 | |
152 | |
153 def Main(): | |
154 urls = FindURLs(AUTOGEN_URL) | |
155 missing = VerifyURLs(urls) | |
156 if missing: | |
157 print 'Found %d Missing files.' % len(missing) | |
158 return 1 | |
159 | |
160 | |
161 if __name__ == '__main__': | |
162 sys.exit(Main()) | |
OLD | NEW |