Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(152)

Side by Side Diff: swarm_get_results.py

Issue 23431002: [Abandoned] Move url_open with dependencies to utils.net module. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/swarm_client
Patch Set: Created 7 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « run_isolated.py ('k') | swarm_trigger_step.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Retrieves all the output that the Swarm server has produced for requests with 6 """Retrieves all the output that the Swarm server has produced for requests with
7 that name. 7 that name.
8 """ 8 """
9 9
10 import json 10 import json
11 import logging 11 import logging
12 import optparse 12 import optparse
13 import sys 13 import sys
14 import time 14 import time
15 import urllib 15 import urllib
16 16
17 from third_party.depot_tools import fix_encoding 17 from third_party.depot_tools import fix_encoding
18 18
19 import run_isolated 19 from utils import net
20
21 from utils import threading_utils 20 from utils import threading_utils
22 from utils import tools 21 from utils import tools
23 22
24 23
25 # The default time to wait for a shard to finish running. 24 # The default time to wait for a shard to finish running.
26 DEFAULT_SHARD_WAIT_TIME = 40 * 60. 25 DEFAULT_SHARD_WAIT_TIME = 40 * 60.
27 26
28 27
29 class Failure(Exception): 28 class Failure(Exception):
30 """Generic failure.""" 29 """Generic failure."""
31 pass 30 pass
32 31
33 32
34 def get_test_keys(swarm_base_url, test_name, _=None): 33 def get_test_keys(swarm_base_url, test_name, _=None):
35 """Returns the Swarm test key for each shards of test_name.""" 34 """Returns the Swarm test key for each shards of test_name."""
36 # TODO(maruel): Remove the parameter '_' once the 35 # TODO(maruel): Remove the parameter '_' once the
37 # build/scripts/slave/get_swarm_results.py stops passing it. 36 # build/scripts/slave/get_swarm_results.py stops passing it.
38 key_data = urllib.urlencode([('name', test_name)]) 37 key_data = urllib.urlencode([('name', test_name)])
39 url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data) 38 url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data)
40 39
41 for i in range(run_isolated.URL_OPEN_MAX_ATTEMPTS): 40 for i in range(net.URL_OPEN_MAX_ATTEMPTS):
42 response = run_isolated.url_open(url, retry_404=True) 41 response = net.url_open(url, retry_404=True)
43 if response is None: 42 if response is None:
44 raise Failure( 43 raise Failure(
45 'Error: Unable to find any tests with the name, %s, on swarm server' 44 'Error: Unable to find any tests with the name, %s, on swarm server'
46 % test_name) 45 % test_name)
47 46
48 result = response.read() 47 result = response.read()
49 # TODO(maruel): Compare exact string. 48 # TODO(maruel): Compare exact string.
50 if 'No matching' in result: 49 if 'No matching' in result:
51 logging.warning('Unable to find any tests with the name, %s, on swarm ' 50 logging.warning('Unable to find any tests with the name, %s, on swarm '
52 'server' % test_name) 51 'server' % test_name)
53 if i != run_isolated.URL_OPEN_MAX_ATTEMPTS: 52 if i != net.URL_OPEN_MAX_ATTEMPTS:
54 run_isolated.HttpService.sleep_before_retry(i, None) 53 net.HttpService.sleep_before_retry(i, None)
55 continue 54 continue
56 return json.loads(result) 55 return json.loads(result)
57 56
58 raise Failure( 57 raise Failure(
59 'Error: Unable to find any tests with the name, %s, on swarm server' 58 'Error: Unable to find any tests with the name, %s, on swarm server'
60 % test_name) 59 % test_name)
61 60
62 61
63 def now(): 62 def now():
64 """Exists so it can be mocked easily.""" 63 """Exists so it can be mocked easily."""
65 return time.time() 64 return time.time()
66 65
67 66
68 def retrieve_results(base_url, test_key, timeout, should_stop): 67 def retrieve_results(base_url, test_key, timeout, should_stop):
69 """Retrieves results for a single test_key.""" 68 """Retrieves results for a single test_key."""
70 assert isinstance(timeout, float) 69 assert isinstance(timeout, float)
71 params = [('r', test_key)] 70 params = [('r', test_key)]
72 result_url = '%s/get_result?%s' % (base_url, urllib.urlencode(params)) 71 result_url = '%s/get_result?%s' % (base_url, urllib.urlencode(params))
73 start = now() 72 start = now()
74 while True: 73 while True:
75 if timeout and (now() - start) >= timeout: 74 if timeout and (now() - start) >= timeout:
76 logging.error('retrieve_results(%s) timed out', base_url) 75 logging.error('retrieve_results(%s) timed out', base_url)
77 return {} 76 return {}
78 # Do retries ourselves. 77 # Do retries ourselves.
79 response = run_isolated.url_open( 78 response = net.url_open(result_url, retry_404=False, retry_50x=False)
80 result_url, retry_404=False, retry_50x=False)
81 if response is None: 79 if response is None:
82 # Aggressively poll for results. Do not use retry_404 so 80 # Aggressively poll for results. Do not use retry_404 so
83 # should_stop is polled more often. 81 # should_stop is polled more often.
84 remaining = min(5, timeout - (now() - start)) if timeout else 5 82 remaining = min(5, timeout - (now() - start)) if timeout else 5
85 if remaining > 0: 83 if remaining > 0:
86 run_isolated.HttpService.sleep_before_retry(1, remaining) 84 net.HttpService.sleep_before_retry(1, remaining)
87 else: 85 else:
88 try: 86 try:
89 data = json.load(response) or {} 87 data = json.load(response) or {}
90 except (ValueError, TypeError): 88 except (ValueError, TypeError):
91 logging.warning( 89 logging.warning(
92 'Received corrupted data for test_key %s. Retrying.', test_key) 90 'Received corrupted data for test_key %s. Retrying.', test_key)
93 else: 91 else:
94 if data['output']: 92 if data['output']:
95 return data 93 return data
96 if should_stop.get(): 94 if should_stop.get():
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
186 output['machine_id'], output['machine_tag'], output['exit_codes'])) 184 output['machine_id'], output['machine_tag'], output['exit_codes']))
187 print(''.join(' %s\n' % l for l in output['output'].splitlines())) 185 print(''.join(' %s\n' % l for l in output['output'].splitlines()))
188 exit_code = max(exit_code, max(map(int, output['exit_codes'].split(',')))) 186 exit_code = max(exit_code, max(map(int, output['exit_codes'].split(','))))
189 187
190 return exit_code 188 return exit_code
191 189
192 190
193 if __name__ == '__main__': 191 if __name__ == '__main__':
194 fix_encoding.fix_encoding() 192 fix_encoding.fix_encoding()
195 sys.exit(main()) 193 sys.exit(main())
OLDNEW
« no previous file with comments | « run_isolated.py ('k') | swarm_trigger_step.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698