OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2017 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 """This script is used to extract network traffic annotations from Chrome. |
| 7 Please refer to README.md for running steps.""" |
| 8 |
| 9 import argparse |
| 10 import os |
| 11 import subprocess |
| 12 import sys |
| 13 |
| 14 # These two lines are required to import protobuf from third_party directory |
| 15 # instead of the one installed with python. |
| 16 from prepare_protobuf import PrepareProtobuf |
| 17 PrepareProtobuf() |
| 18 |
| 19 from google.protobuf import text_format |
| 20 import traffic_annotation_pb2 |
| 21 |
| 22 |
| 23 def _RunClangTool(src_dir, build_dir, path_filters): |
| 24 """Executes the clang tool to extract annotations. |
| 25 Args: |
| 26 src_dir: str Path to the src directory of Chrome. |
| 27 build_dir: str Path to the build directory. |
| 28 path_filters: list of str List of paths to source directories for |
| 29 extraction. |
| 30 |
| 31 Returns: |
| 32 raw_annotations: str Output of clang tool (extracted content and metadata of |
| 33 annotations). |
| 34 """ |
| 35 raw_annotations = "" |
| 36 for path in path_filters: |
| 37 args = [ |
| 38 src_dir + "/tools/clang/scripts/run_tool.py", |
| 39 "--generate-compdb", |
| 40 "traffic_annotation_extractor", |
| 41 build_dir, path] |
| 42 if sys.platform == "win32": |
| 43 args.insert(0, "python") |
| 44 command = subprocess.Popen(args, stdout=subprocess.PIPE, |
| 45 stderr=subprocess.PIPE) |
| 46 stdout_text, stderr_text = command.communicate() |
| 47 raw_annotations += stdout_text |
| 48 if stderr_text: |
| 49 print stderr_text |
| 50 return raw_annotations |
| 51 |
| 52 |
| 53 def _ParsRawAnnotations(raw_annotations): |
| 54 """Parses raw annotations texts which are received from the clang tool. |
| 55 Args: |
| 56 raw_annotations: str Serialization of annotations and metadata. Each |
| 57 annotation should have the following lines: |
| 58 1- "==== NEW ANNOTATION ====" |
| 59 2- File path. |
| 60 3- Name of the function including this position. |
| 61 4- Line number. |
| 62 5- Unique id of annotation. |
| 63 6- Serialization of annotation text (several lines) |
| 64 n- "==== ANNOTATION ENDS ====" |
| 65 |
| 66 Returns: |
| 67 annotations: ExtractedNetworkTrafficAnnotation A protobuf including all |
| 68 extracted annotations. |
| 69 errors: list of str List of errors. |
| 70 """ |
| 71 annotations = traffic_annotation_pb2.ExtractedNetworkTrafficAnnotation() |
| 72 errors = [] |
| 73 |
| 74 lines = [line.strip("\r\n") for line in raw_annotations.split("\n")] |
| 75 current = 0 |
| 76 |
| 77 try: |
| 78 while current < len(lines) - 1: |
| 79 if lines[current] != "==== NEW ANNOTATION ====": |
| 80 raise Exception( |
| 81 "Error at line %i, expected starting new annotaion." % current) |
| 82 if current + 5 >= len(lines): |
| 83 raise Exception( |
| 84 "Not enough header lines at line %i." % current) |
| 85 |
| 86 # Extract header lines. |
| 87 source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource() |
| 88 source.file = lines[current + 1] |
| 89 source.function = lines[current + 2] |
| 90 source.line = int(lines[current + 3]) |
| 91 unique_id = lines[current + 4] |
| 92 |
| 93 # Extract serialized proto. |
| 94 current += 5 |
| 95 annotation_text = "" |
| 96 |
| 97 while current < len(lines): |
| 98 current += 1 |
| 99 if lines[current - 1] == "==== ANNOTATION ENDS ====": |
| 100 break |
| 101 else: |
| 102 annotation_text += lines[current - 1] |
| 103 else: |
| 104 raise Exception( |
| 105 "Error at line %i, expected annotation end tag." % current) |
| 106 |
| 107 # Process unittests and undefined tags. |
| 108 if unique_id == "UnitTest": |
| 109 continue |
| 110 if unique_id == "Undefined": |
| 111 errors.append("Annotation is not defined for file '%s', line %i." % |
| 112 (source.file, source.line)) |
| 113 continue |
| 114 |
| 115 # Decode serialized proto. |
| 116 annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation() |
| 117 try: |
| 118 text_format.Parse(annotation_text, annotation_proto) |
| 119 except Exception as error: |
| 120 errors.append("Annotation in file '%s', line %i, has error: %s" % |
| 121 (source.file, source.line, error)) |
| 122 |
| 123 # Add new proto. |
| 124 annotation_proto.unique_id = unique_id |
| 125 annotation_proto.source.CopyFrom(source) |
| 126 annotations.network_traffic_annotation.add().CopyFrom(annotation_proto) |
| 127 |
| 128 except Exception as error: |
| 129 errors.append(str(error)) |
| 130 |
| 131 print "Extracted %i annotations with %i errors." % \ |
| 132 (len(annotations.network_traffic_annotation), len(errors)) |
| 133 return annotations, errors |
| 134 |
| 135 |
| 136 def _WriteSummaryFile(annotations, errors, file_path): |
| 137 """Writes extracted annotations and errors into a simple text file. |
| 138 args: |
| 139 annotations ExtractedNetworkTrafficAnnotation A protobuf including all |
| 140 extracted annotations. |
| 141 errors list of str List of all extraction errors. |
| 142 file_path str File path to the brief summary file. |
| 143 """ |
| 144 with open(file_path, 'w') as summary_file: |
| 145 if errors: |
| 146 summary_file.write("Errors:\n%s\n\n" % "\n".join(errors)) |
| 147 if len(annotations.network_traffic_annotation): |
| 148 summary_file.write("Annotations:\n%s" % "\n---\n".join( |
| 149 [str(a) for a in annotations.network_traffic_annotation])) |
| 150 |
| 151 |
| 152 def main(): |
| 153 parser = argparse.ArgumentParser(description='Traffic Annotation Auditor.') |
| 154 parser.add_argument('--build-dir', |
| 155 help='Path to the build directory.') |
| 156 parser.add_argument('--extractor-output', |
| 157 help='Optional path to the temporary file that extracted ' |
| 158 'annotations will be stored into.') |
| 159 parser.add_argument('--extractor-input', |
| 160 help='Optional path to the file that temporary extracted ' |
| 161 'annotations are already stored in. If this is ' |
| 162 'provided, clang tool is not run and this is used ' |
| 163 'as input.') |
| 164 parser.add_argument('--summary-file', |
| 165 help='Path to the output file.') |
| 166 parser.add_argument('path_filters', |
| 167 nargs='*', |
| 168 help='Optional paths to filter what files the tool is ' |
| 169 'run on.') |
| 170 args = parser.parse_args() |
| 171 |
| 172 if not args.summary_file: |
| 173 print "Warning: Output file not specified." |
| 174 |
| 175 # If a pre-extracted input file is provided, load it. |
| 176 if args.extractor_input: |
| 177 with open(args.extractor_input, 'r') as raw_file: |
| 178 raw_annotations = raw_file.read() |
| 179 else: |
| 180 # Either extacted input file or build directory should be provided. |
| 181 if not args.build_dir: |
| 182 print "You must either specify the build directory to run the clang " \ |
| 183 "tool and extract annotations, or specify the input directory " \ |
| 184 "where extracted annotation files already exist.\n" |
| 185 return 1 |
| 186 |
| 187 # Get Chrome source directory with relative path from this file. |
| 188 chrome_source = os.path.abspath(os.path.join(os.path.dirname( |
| 189 os.path.realpath(__file__)), "..", "..", "..")) |
| 190 raw_annotations = _RunClangTool(chrome_source, args.build_dir, |
| 191 args.path_filters if args.path_filters else ["./"]) |
| 192 |
| 193 if args.extractor_output: |
| 194 with open(args.extractor_output, 'w') as raw_file: |
| 195 raw_file.write(raw_annotations) |
| 196 |
| 197 annotations, errors = _ParsRawAnnotations(raw_annotations) |
| 198 |
| 199 if not annotations: |
| 200 print "Could not extract any annotation." |
| 201 if errors: |
| 202 print "Errors:\n%s" % "\n".join(errors) |
| 203 return 1 |
| 204 |
| 205 if args.summary_file: |
| 206 _WriteSummaryFile(annotations, errors, args.summary_file) |
| 207 |
| 208 return 0 |
| 209 |
| 210 |
| 211 if __name__ == '__main__': |
| 212 sys.exit(main()) |
OLD | NEW |