OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2017 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2017 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """This script is used to extract network traffic annotations from Chrome. | 6 """This script is used to extract network traffic annotations from Chrome. |
7 Please refer to README.md for running steps.""" | 7 Please refer to README.md for running steps.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import os | 10 import os |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
97 """ | 97 """ |
98 annotations = traffic_annotation_pb2.ExtractedNetworkTrafficAnnotation() | 98 annotations = traffic_annotation_pb2.ExtractedNetworkTrafficAnnotation() |
99 errors = [] | 99 errors = [] |
100 metadata = [] | 100 metadata = [] |
101 | 101 |
102 lines = [line.strip("\r\n") for line in raw_annotations.split("\n")] | 102 lines = [line.strip("\r\n") for line in raw_annotations.split("\n")] |
103 current = 0 | 103 current = 0 |
104 | 104 |
105 try: | 105 try: |
106 while current < len(lines) - 1: | 106 while current < len(lines) - 1: |
107 if lines[current] != "==== NEW ANNOTATION ====": | 107 if lines[current] == "==== NEW ANNOTATION ====": |
| 108 if current + 5 >= len(lines): |
| 109 raise Exception( |
| 110 "Not enough header lines at line %i." % current) |
| 111 |
| 112 # Extract header lines. |
| 113 source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource() |
| 114 source.file = lines[current + 1] |
| 115 source.function = lines[current + 2] |
| 116 source.line = int(lines[current + 3]) |
| 117 unique_id = lines[current + 5] |
| 118 |
| 119 new_metadata = {'function_type': lines[current + 4], |
| 120 'extra_id': lines[current + 6], |
| 121 'unique_id_hash': _ComputeStringHash(unique_id)} |
| 122 # Extract serialized proto. |
| 123 current += 7 |
| 124 annotation_text = "" |
| 125 |
| 126 while current < len(lines): |
| 127 current += 1 |
| 128 if lines[current - 1] == "==== ANNOTATION ENDS ====": |
| 129 break |
| 130 else: |
| 131 annotation_text += lines[current - 1] |
| 132 else: |
| 133 raise Exception( |
| 134 "Error at line %i, expected annotation end tag." % current) |
| 135 |
| 136 # Process unittests and undefined tags. |
| 137 if unique_id in ("test", "test_partial"): |
| 138 continue |
| 139 if unique_id in ("undefined", "missing"): |
| 140 errors.append("Annotation is not defined for file '%s', line %i." % |
| 141 (source.file, source.line)) |
| 142 continue |
| 143 |
| 144 # Decode serialized proto. |
| 145 annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation() |
| 146 try: |
| 147 text_format.Parse(annotation_text, annotation_proto) |
| 148 except Exception as error: |
| 149 errors.append("Annotation in file '%s', line %i, has error: %s" % |
| 150 (source.file, source.line, error)) |
| 151 |
| 152 # Add new proto. |
| 153 annotation_proto.unique_id = unique_id |
| 154 annotation_proto.source.CopyFrom(source) |
| 155 annotations.network_traffic_annotation.add().CopyFrom(annotation_proto) |
| 156 metadata.append(new_metadata) |
| 157 elif lines[current] == "==== NEW CALL ====": |
| 158 # Ignore calls for now. |
| 159 while current < len(lines): |
| 160 current += 1 |
| 161 if lines[current - 1] == "==== CALL ENDS ====": |
| 162 break |
| 163 else: |
| 164 raise Exception( |
| 165 "Error at line %i, expected call end tag." % current) |
| 166 else: # The line is neither new annotation nor new call. |
108 raise Exception( | 167 raise Exception( |
109 "Error at line %i, expected starting new annotaion." % current) | 168 "Error at line %i, expected starting new annotation or call." % |
110 if current + 5 >= len(lines): | 169 current) |
111 raise Exception( | |
112 "Not enough header lines at line %i." % current) | |
113 | |
114 # Extract header lines. | |
115 source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource() | |
116 source.file = lines[current + 1] | |
117 source.function = lines[current + 2] | |
118 source.line = int(lines[current + 3]) | |
119 unique_id = lines[current + 5] | |
120 | |
121 new_metadata = {'function_type': lines[current + 4], | |
122 'extra_id': lines[current + 6], | |
123 'unique_id_hash': _ComputeStringHash(unique_id)} | |
124 # Extract serialized proto. | |
125 current += 7 | |
126 annotation_text = "" | |
127 | |
128 while current < len(lines): | |
129 current += 1 | |
130 if lines[current - 1] == "==== ANNOTATION ENDS ====": | |
131 break | |
132 else: | |
133 annotation_text += lines[current - 1] | |
134 else: | |
135 raise Exception( | |
136 "Error at line %i, expected annotation end tag." % current) | |
137 | |
138 # Process unittests and undefined tags. | |
139 if unique_id in ("test", "test_partial"): | |
140 continue | |
141 if unique_id in ("undefined", "missing"): | |
142 errors.append("Annotation is not defined for file '%s', line %i." % | |
143 (source.file, source.line)) | |
144 continue | |
145 | |
146 # Decode serialized proto. | |
147 annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation() | |
148 try: | |
149 text_format.Parse(annotation_text, annotation_proto) | |
150 except Exception as error: | |
151 errors.append("Annotation in file '%s', line %i, has error: %s" % | |
152 (source.file, source.line, error)) | |
153 | |
154 # Add new proto. | |
155 annotation_proto.unique_id = unique_id | |
156 annotation_proto.source.CopyFrom(source) | |
157 annotations.network_traffic_annotation.add().CopyFrom(annotation_proto) | |
158 metadata.append(new_metadata) | |
159 | 170 |
160 except Exception as error: | 171 except Exception as error: |
161 errors.append(str(error)) | 172 errors.append(str(error)) |
162 | 173 |
163 print "Extracted %i annotations with %i errors." % \ | 174 print "Extracted %i annotations with %i errors." % \ |
164 (len(annotations.network_traffic_annotation), len(errors)) | 175 (len(annotations.network_traffic_annotation), len(errors)) |
165 return annotations, metadata, errors | 176 return annotations, metadata, errors |
166 | 177 |
167 | 178 |
168 def _WriteSummaryFile(annotations, metadata, errors, file_path): | 179 def _WriteSummaryFile(annotations, metadata, errors, file_path): |
(...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
265 _WriteSummaryFile(annotations, metadata, errors, args.summary_file) | 276 _WriteSummaryFile(annotations, metadata, errors, args.summary_file) |
266 | 277 |
267 if args.hash_codes_file: | 278 if args.hash_codes_file: |
268 _WriteHashCodesFile(annotations, metadata, args.hash_codes_file) | 279 _WriteHashCodesFile(annotations, metadata, args.hash_codes_file) |
269 | 280 |
270 return 0 | 281 return 0 |
271 | 282 |
272 | 283 |
273 if __name__ == '__main__': | 284 if __name__ == '__main__': |
274 sys.exit(main()) | 285 sys.exit(main()) |
OLD | NEW |