Tool added to extract network traffic annotations.

tools/traffic_annotation/auditor/traffic_annotation_auditor.py

+#!/usr/bin/env python
+# Copyright (c) 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This script is used to network extract traffic annotations from Chromium.
+"""
+
+import argparse
+# import google.protobuf.text_format
+import os
+import subprocess
+import sys
+
+# out/Debug/protoc --python_out=tools/traffic_annotation/auditor
+#  --proto_path=out/Debug/gen/components/policy/proto --proto_path=
+# tools/traffic_annotation tools/traffic_annotation/traffic_annotation.proto
+
+#import traffic_annotation_pb2
+# import distutils.spawn
+# import glob
+# import os
+# import pipes
+# import re
+# import shutil
+# 
+# import stat
+# 
+# import tarfile
+# import tempfile
+# import time
+# import urllib2
+# import zipfile
+
+
+def _RunClangTool(src_dir, build_dir, path_filters):
+  """Executes the clang tool too extract annotations.
+  Args:
+    src_dir: str Path to the src directory of Chromium.
+    build_dir: str Path to the build directory.
+    path_filters: list of str List of paths to source directories for
+        extraction.
+
+  Returns:
+    raw_annotaions: str Extracted content and meta data of annotations.
+  """
+  raw_annotations = ""
+  for path in path_filters:
+    args = [
+        src_dir + "/tools/clang/scripts/run_tool.py",
+        "--generate-compdb",
+        "traffic_annotation_extractor",
+        build_dir, path]
+    command = subprocess.Popen(args, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout_text, stderr_text = command.communicate()
+    raw_annotations += stdout_text
+    if stderr_text:
+      print stderr_text
+  return raw_annotations
+
+
+def _ParsRawAnnotations(raw_annotations):
+  """Parses raw annotations texts.
+  Args:
+    raw_annotations: str Serialization of annotations and meta data. Each
+        annotation should have the following lines:
+        1- ==== NEW ANNOTATION ====
+        2- File path.
+        3- Name of the function including this position.
+        4- Line number.
+        5- Unique id of annotation.
+        6- Serialization of annotation text (several lines)
+        n- ==== ANNOTATION ENDS ====
+
+  Returns:
+    annotations: list of dict List of dictionaries including annotations and
+        meta data.
+    errors: list of dict List of errors.
+  """
+  annotations = []
+  errors = []
+
+  lines = raw_annotations.split("\n")
+  current = 0
+
+  try:
+    while current < len(lines) - 1:
+      if lines[current] != "==== NEW ANNOTATION ====":
+        raise Exception(
+          "Error at line %i, expected starting new annotaion." % current)
+      if current + 5 >= len(lines):
+        raise Exception(
+          "Not enough header lines at line %i." % current)
+
+      new_annotation = {'file_path': lines[current + 1],
+                        'function_name': lines[current + 2],
+                        'line_number': lines[current + 3],
+                        'unique_id': lines[current + 4]}
+      current += 5
+      annotation_text = ""
+      
+      while current < len(lines):
+        current += 1
+        if lines[current - 1] == "==== ANNOTATION ENDS ====":        
+          break
+        else:
+          annotation_text += lines[current - 1]
+      else:
+        raise Exception(
+          "Error at line %i, expected annotation end tag." % current)
+
+      # Parse annotation text.
+      annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotations()
+            
+
+      new_annotation['annotation_text'] = annotation_text
+      annotations.append(new_annotation)
+  except Exception as error:
+    print error
+    return [], []
+  
+  print annotations
+  
+  return annotations, errors
+
+
+def main():
+  parser = argparse.ArgumentParser(description='Traffic Annotation Auditor.')
+  parser.add_argument('--build-dir', 
+                      help='Path to the build directory.')
+  parser.add_argument('--extractor-output',
+                      help='Optional path to the temporary file that extracted '
+                           'annotations will be stored.')
+  parser.add_argument('--extractor-input',
+                      help='Optional Path to the file that temporary extracted '
+                           'annotations are already stored in. If this is '
+                           'provided, clang tool is not run and this is used '
+                           'as input.')
+  parser.add_argument('--summary-file',
+                      help='Path to the output file.')
+  parser.add_argument('path_filters',
+                      nargs='*',
+                      help='optional paths to filter what files the tool is '
+                           'run on.')
+  args = parser.parse_args()
+
+  if not args.summary_file:
+    print "Warning: Output file not specified."
+
+  if args.extractor_input:
+    with open(args.extractor_input, 'r') as raw_file:
+      raw_annotations = raw_file.read()
+  else:
+    # Either extacted input file or build directory should be provided.
+    if not args.build_dir:
+      print "You must either specify the build directory to run the clang " \
+            "tool and extract annotations, or specify the input directory " \
+            "where extracted annotation files already exist.\n"
+      return
+
+    # Output diretory is usually two steps in src directory, so the path to 
+    # source is extracted by moving two directories up.
+    raw_annotations = _RunClangTool(
+        args.build_dir + "/../..", args.build_dir,
+        args.path_filters if args.path_filters else ["./"])   
+
+  if args.extractor_output:
+    with open(args.extractor_output, 'w') as raw_file:
+      raw_file.write(raw_annotations)    
+    
+  _ParsRawAnnotations(raw_annotations)
+  
+
+  # if (instances.empty()) {
+  #   LOG(ERROR) << "Could not read any file.\n";
+  #   return 1;
+  # } else {
+  #   MarkRepeatedUniqueIds(&instances);
+  # }
+
+  # // Create Summary file if requested.
+  # if (!summary_file.empty() &&
+  #     !WriteSummaryFile(clang_tool_exit_code, instances, errors, summary_file))
+  #   return 1;
+
+
+
+
+
+if __name__ == '__main__':
+  sys.exit(main())