OLD | NEW |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 # Recipe for uploading DM results. | 6 # Recipe for uploading DM results. |
7 | 7 |
8 | 8 |
9 DEPS = [ | 9 DEPS = [ |
10 'build/file', | 10 'build/file', |
11 'recipe_engine/json', | 11 'recipe_engine/json', |
12 'recipe_engine/path', | 12 'recipe_engine/path', |
13 'recipe_engine/properties', | 13 'recipe_engine/properties', |
14 'recipe_engine/shutil', | 14 'recipe_engine/shutil', |
15 'recipe_engine/step', | 15 'recipe_engine/step', |
16 'recipe_engine/time', | 16 'recipe_engine/time', |
17 ] | 17 ] |
18 | 18 |
19 | 19 |
20 import time | 20 import time |
21 | 21 |
22 | 22 |
23 DM_JSON = 'dm.json' | 23 DM_JSON = 'dm.json' |
24 GS_BUCKET = 'gs://skia-infra-gm' | 24 GS_BUCKET = 'gs://skia-infra-gm' |
| 25 UPLOAD_ATTEMPTS = 5 |
25 VERBOSE_LOG = 'verbose.log' | 26 VERBOSE_LOG = 'verbose.log' |
26 | 27 |
27 | 28 |
| 29 def cp(api, name, src, dst, extra_args=None): |
| 30 cmd = ['gsutil', 'cp'] |
| 31 if extra_args: |
| 32 cmd.extend(extra_args) |
| 33 cmd.extend([src, dst]) |
| 34 |
| 35 name = 'upload %s' % name |
| 36 for i in xrange(UPLOAD_ATTEMPTS): |
| 37 step_name = name |
| 38 if i > 0: |
| 39 step_name += ' (attempt %d)' % (i+1) |
| 40 try: |
| 41 api.step(step_name, cmd=cmd) |
| 42 break |
| 43 except api.step.StepFailure: |
| 44 if i == UPLOAD_ATTEMPTS - 1: |
| 45 raise |
| 46 |
| 47 |
28 def RunSteps(api): | 48 def RunSteps(api): |
29 builder_name = api.properties['buildername'] | 49 builder_name = api.properties['buildername'] |
30 revision = api.properties['revision'] | 50 revision = api.properties['revision'] |
31 | 51 |
32 patch_storage = api.properties.get('patch_storage', 'rietveld') | 52 patch_storage = api.properties.get('patch_storage', 'rietveld') |
33 issue = None | 53 issue = None |
34 patchset = None | 54 patchset = None |
35 if builder_name.endswith('-Trybot'): | 55 if builder_name.endswith('-Trybot'): |
36 if patch_storage == 'gerrit': | 56 if patch_storage == 'gerrit': |
37 issue = str(api.properties['event.change.number']) | 57 issue = str(api.properties['event.change.number']) |
(...skipping 15 matching lines...) Expand all Loading... |
53 api.shutil.remove('rm old verbose.log', log_file) | 73 api.shutil.remove('rm old verbose.log', log_file) |
54 | 74 |
55 # Upload the images. | 75 # Upload the images. |
56 image_dest_path = '/'.join((GS_BUCKET, 'dm-images-v1')) | 76 image_dest_path = '/'.join((GS_BUCKET, 'dm-images-v1')) |
57 files_to_upload = api.file.glob( | 77 files_to_upload = api.file.glob( |
58 'find images', | 78 'find images', |
59 results_dir.join('*'), | 79 results_dir.join('*'), |
60 test_data=['someimage.png'], | 80 test_data=['someimage.png'], |
61 infra_step=True) | 81 infra_step=True) |
62 if len(files_to_upload) > 0: | 82 if len(files_to_upload) > 0: |
63 api.step( | 83 cp(api, 'images', results_dir.join('*'), image_dest_path) |
64 'upload images', | |
65 cmd=['gsutil', 'cp', results_dir.join('*'), image_dest_path], | |
66 ) | |
67 | 84 |
68 # Upload the JSON summary and verbose.log. | 85 # Upload the JSON summary and verbose.log. |
69 now = api.time.utcnow() | 86 now = api.time.utcnow() |
70 summary_dest_path = '/'.join([ | 87 summary_dest_path = '/'.join([ |
71 'dm-json-v1', | 88 'dm-json-v1', |
72 str(now.year ).zfill(4), | 89 str(now.year ).zfill(4), |
73 str(now.month).zfill(2), | 90 str(now.month).zfill(2), |
74 str(now.day ).zfill(2), | 91 str(now.day ).zfill(2), |
75 str(now.hour ).zfill(2), | 92 str(now.hour ).zfill(2), |
76 revision, | 93 revision, |
77 builder_name, | 94 builder_name, |
78 str(int(time.mktime(now.utctimetuple())))]) | 95 str(int(time.mktime(now.utctimetuple())))]) |
79 | 96 |
80 # Trybot results are further siloed by issue/patchset. | 97 # Trybot results are further siloed by issue/patchset. |
81 if builder_name.endswith('-Trybot'): | 98 if builder_name.endswith('-Trybot'): |
82 if not (issue and patchset): # pragma: nocover | 99 if not (issue and patchset): # pragma: nocover |
83 raise Exception('issue and patchset properties are required for trybots.') | 100 raise Exception('issue and patchset properties are required for trybots.') |
84 summary_dest_path = '/'.join(('trybot', summary_dest_path, issue, patchset)) | 101 summary_dest_path = '/'.join(('trybot', summary_dest_path, issue, patchset)) |
85 | 102 |
86 summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path)) | 103 summary_dest_path = '/'.join((GS_BUCKET, summary_dest_path)) |
87 | 104 |
88 api.step( | 105 cp(api, 'JSON and logs', tmp_dir.join('*'), summary_dest_path, |
89 'upload JSON and logs', | 106 ['-z', 'json,log']) |
90 cmd=['gsutil', 'cp', '-z', 'json,log', tmp_dir.join('*'), | |
91 summary_dest_path], | |
92 ) | |
93 | 107 |
94 | 108 |
95 def GenTests(api): | 109 def GenTests(api): |
96 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' | 110 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug' |
97 yield ( | 111 yield ( |
98 api.test('normal_bot') + | 112 api.test('normal_bot') + |
99 api.properties(buildername=builder, | 113 api.properties(buildername=builder, |
100 revision='abc123', | 114 revision='abc123', |
101 path_config='kitchen') | 115 path_config='kitchen') |
102 ) | 116 ) |
103 | 117 |
| 118 yield ( |
| 119 api.test('failed_once') + |
| 120 api.properties(buildername=builder, |
| 121 revision='abc123', |
| 122 path_config='kitchen') + |
| 123 api.step_data('upload images', retcode=1) |
| 124 ) |
| 125 |
| 126 yield ( |
| 127 api.test('failed_all') + |
| 128 api.properties(buildername=builder, |
| 129 revision='abc123', |
| 130 path_config='kitchen') + |
| 131 api.step_data('upload images', retcode=1) + |
| 132 api.step_data('upload images (attempt 2)', retcode=1) + |
| 133 api.step_data('upload images (attempt 3)', retcode=1) + |
| 134 api.step_data('upload images (attempt 4)', retcode=1) + |
| 135 api.step_data('upload images (attempt 5)', retcode=1) |
| 136 ) |
| 137 |
104 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot' | 138 builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot' |
105 yield ( | 139 yield ( |
106 api.test('trybot') + | 140 api.test('trybot') + |
107 api.properties(buildername=builder, | 141 api.properties(buildername=builder, |
108 revision='abc123', | 142 revision='abc123', |
109 path_config='kitchen', | 143 path_config='kitchen', |
110 issue='12345', | 144 issue='12345', |
111 patchset='1002') | 145 patchset='1002') |
112 ) | 146 ) |
113 | 147 |
114 gerrit_kwargs = { | 148 gerrit_kwargs = { |
115 'patch_storage': 'gerrit', | 149 'patch_storage': 'gerrit', |
116 'repository': 'skia', | 150 'repository': 'skia', |
117 'event.patchSet.ref': 'refs/changes/00/2100/2', | 151 'event.patchSet.ref': 'refs/changes/00/2100/2', |
118 'event.change.number': '2100', | 152 'event.change.number': '2100', |
119 } | 153 } |
120 yield ( | 154 yield ( |
121 api.test('recipe_with_gerrit_patch') + | 155 api.test('recipe_with_gerrit_patch') + |
122 api.properties( | 156 api.properties( |
123 buildername=builder, | 157 buildername=builder, |
124 revision='abc123', | 158 revision='abc123', |
125 path_config='kitchen', | 159 path_config='kitchen', |
126 **gerrit_kwargs) | 160 **gerrit_kwargs) |
127 ) | 161 ) |
OLD | NEW |