Expanding traffic_annotation_extractor clang tool to extract network calls.

tools/traffic_annotation/auditor/traffic_annotation_auditor.py

 #!/usr/bin/env python
 # Copyright (c) 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """This script is used to extract network traffic annotations from Chrome.
 Please refer to README.md for running steps."""
 
 import argparse
 import os
@@ skipping 56 matching lines @@
     stdout_text, stderr_text = command.communicate()
     raw_annotations += stdout_text
     if stderr_text:
       print stderr_text
   return raw_annotations
 
 
 def _ParsRawAnnotations(raw_annotations):
   """Parses raw annotations texts which are received from the clang tool.
   Args:
     raw_annotations: str Serialization of annotations and metadata. Each

[msramek, 2017/05/29 21:03:37]

Please update the function doc to mention calls as well.

[Ramin Halavati, 2017/05/30 04:40:37]

Done.

         annotation should have the following lines:
         1- "==== NEW ANNOTATION ===="
         2- File path.
         3- Name of the function including this position.
         4- Line number.
         5- Function Type.
         6- Unique id of annotation.
         7- Completing id or group id, when applicable, empty otherwise.
         8- Serialization of annotation text (several lines)
         n- "==== ANNOTATION ENDS ===="
 
   Returns:
     annotations: ExtractedNetworkTrafficAnnotation A protobuf including all
       extracted annotations.
     metadata: list of dict List of metadata for each annotation. Each item
       includes the following fields:
         function_type: str Type of the function that defines the annotation.
         extra_id: str Possible prefix for annotation completion.
     errors: list of str List of errors.
   """
   annotations = traffic_annotation_pb2.ExtractedNetworkTrafficAnnotation()
   errors = []
   metadata = []
 
   lines = [line.strip("\r\n") for line in raw_annotations.split("\n")]
   current = 0
 
   try:
     while current < len(lines) - 1:

[msramek, 2017/05/29 21:03:37]

Not necessarily in this CL, but this logic looks like a good candidate for a
unittest. We can have a similar setup as in our team's automation scripts.

[Ramin Halavati, 2017/05/30 04:40:37]

Acknowledged.

-      if lines[current] != "==== NEW ANNOTATION ====":
+      if lines[current] == "==== NEW ANNOTATION ====":
+        if current + 5 >= len(lines):

[msramek, 2017/05/29 21:03:36]

+5 is not enough, since you're accessing +6 at line 120.

[Ramin Halavati, 2017/05/30 04:40:37]

Done.

+          raise Exception(
+              "Not enough header lines at line %i." % current)
+
+        # Extract header lines.
+        source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource()
+        source.file = lines[current + 1]
+        source.function = lines[current + 2]
+        source.line = int(lines[current + 3])
+        unique_id = lines[current + 5]
+
+        new_metadata = {'function_type': lines[current + 4],
+                        'extra_id': lines[current + 6],
+                        'unique_id_hash': _ComputeStringHash(unique_id)}
+        # Extract serialized proto.
+        current += 7
+        annotation_text = ""
+
+        while current < len(lines):
+          current += 1

[msramek, 2017/05/29 21:03:37]

optional: Consider moving this current += 1 at the end of the body of the large
while statement? I think it's a bit more readable when we just make this +1 step
to the next row at the end rather than always having to do [-1]. Possibly also
add a comment why is that += 1 there.

[Ramin Halavati, 2017/05/30 04:40:36]

Done.

+          if lines[current - 1] == "==== ANNOTATION ENDS ====":
+            break
+          else:
+            annotation_text += lines[current - 1]
+        else:
+          raise Exception(
+            "Error at line %i, expected annotation end tag." % current)
+

[msramek, 2017/05/29 21:03:37]

Should we handle an empty |annotation_text| here?

[Ramin Halavati, 2017/05/30 04:40:36]

It would be checked later when proto is deserialized. I will add it to unittest
so that it ensures that it's tested somewhere.

+        # Process unittests and undefined tags.
+        if unique_id in ("test", "test_partial"):
+          continue
+        if unique_id in ("undefined", "missing"):
+          errors.append("Annotation is not defined for file '%s', line %i." %
+              (source.file, source.line))
+          continue
+
+        # Decode serialized proto.
+        annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation()
+        try:
+          text_format.Parse(annotation_text, annotation_proto)
+        except Exception as error:
+          errors.append("Annotation in file '%s', line %i, has error: %s" %

[msramek, 2017/05/29 21:03:37]

nit: an error

[Ramin Halavati, 2017/05/30 04:40:36]

Done.

+              (source.file, source.line, error))
+
+        # Add new proto.
+        annotation_proto.unique_id = unique_id
+        annotation_proto.source.CopyFrom(source)
+        annotations.network_traffic_annotation.add().CopyFrom(annotation_proto)
+        metadata.append(new_metadata)
+      elif lines[current] == "==== NEW CALL ====":
+        # Ignore calls for now.
+        while current < len(lines):
+          current += 1
+          if lines[current - 1] == "==== CALL ENDS ====":
+            break
+        else:
+          raise Exception(
+              "Error at line %i, expected call end tag." % current)
+      else: # The line is neither new annotation nor new call.
         raise Exception(
-            "Error at line %i, expected starting new annotaion." % current)
-      if current + 5 >= len(lines):
-        raise Exception(
-            "Not enough header lines at line %i." % current)
-
-      # Extract header lines.
-      source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource()
-      source.file = lines[current + 1]
-      source.function = lines[current + 2]
-      source.line = int(lines[current + 3])
-      unique_id = lines[current + 5]
-
-      new_metadata = {'function_type': lines[current + 4],
-                      'extra_id': lines[current + 6],
-                      'unique_id_hash': _ComputeStringHash(unique_id)}
-      # Extract serialized proto.
-      current += 7
-      annotation_text = ""
-
-      while current < len(lines):
-        current += 1
-        if lines[current - 1] == "==== ANNOTATION ENDS ====":
-          break
-        else:
-          annotation_text += lines[current - 1]
-      else:
-        raise Exception(
-          "Error at line %i, expected annotation end tag." % current)
-
-      # Process unittests and undefined tags.
-      if unique_id in ("test", "test_partial"):
-        continue
-      if unique_id in ("undefined", "missing"):
-        errors.append("Annotation is not defined for file '%s', line %i." %
-            (source.file, source.line))
-        continue
-
-      # Decode serialized proto.
-      annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation()
-      try:
-        text_format.Parse(annotation_text, annotation_proto)
-      except Exception as error:
-        errors.append("Annotation in file '%s', line %i, has error: %s" %
-            (source.file, source.line, error))
-
-      # Add new proto.
-      annotation_proto.unique_id = unique_id
-      annotation_proto.source.CopyFrom(source)
-      annotations.network_traffic_annotation.add().CopyFrom(annotation_proto)
-      metadata.append(new_metadata)
+            "Error at line %i, expected starting new annotation or call." %
+            current)
 
   except Exception as error:
     errors.append(str(error))
 
   print "Extracted %i annotations with %i errors." % \
     (len(annotations.network_traffic_annotation), len(errors))
   return annotations, metadata, errors
 
 
 def _WriteSummaryFile(annotations, metadata, errors, file_path):
@@ skipping 96 matching lines @@
     _WriteSummaryFile(annotations, metadata, errors, args.summary_file)
 
   if args.hash_codes_file:
     _WriteHashCodesFile(annotations, metadata, args.hash_codes_file)
 
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())