Expanding traffic_annotation_extractor clang tool to extract network calls.

tools/traffic_annotation/auditor/traffic_annotation_auditor.py

Index: tools/traffic_annotation/auditor/traffic_annotation_auditor.py
diff --git a/tools/traffic_annotation/auditor/traffic_annotation_auditor.py b/tools/traffic_annotation/auditor/traffic_annotation_auditor.py
index 56e82d99b3d8ef4c607d2c484986221b2f35d024..20100ade93747616a395fd9dad46b234fde0d209 100755
--- a/tools/traffic_annotation/auditor/traffic_annotation_auditor.py
+++ b/tools/traffic_annotation/auditor/traffic_annotation_auditor.py
@@ -104,58 +104,69 @@ def _ParsRawAnnotations(raw_annotations):
 
   try:
     while current < len(lines) - 1:

[msramek, 2017/05/29 21:03:37]

Not necessarily in this CL, but this logic looks like a good candidate for a
unittest. We can have a similar setup as in our team's automation scripts.

[Ramin Halavati, 2017/05/30 04:40:37]

Acknowledged.

-      if lines[current] != "==== NEW ANNOTATION ====":
-        raise Exception(
-            "Error at line %i, expected starting new annotaion." % current)
-      if current + 5 >= len(lines):
-        raise Exception(
-            "Not enough header lines at line %i." % current)
-
-      # Extract header lines.
-      source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource()
-      source.file = lines[current + 1]
-      source.function = lines[current + 2]
-      source.line = int(lines[current + 3])
-      unique_id = lines[current + 5]
-
-      new_metadata = {'function_type': lines[current + 4],
-                      'extra_id': lines[current + 6],
-                      'unique_id_hash': _ComputeStringHash(unique_id)}
-      # Extract serialized proto.
-      current += 7
-      annotation_text = ""
-
-      while current < len(lines):
-        current += 1
-        if lines[current - 1] == "==== ANNOTATION ENDS ====":
-          break
+      if lines[current] == "==== NEW ANNOTATION ====":
+        if current + 5 >= len(lines):

[msramek, 2017/05/29 21:03:36]

+5 is not enough, since you're accessing +6 at line 120.

[Ramin Halavati, 2017/05/30 04:40:37]

Done.

+          raise Exception(
+              "Not enough header lines at line %i." % current)
+
+        # Extract header lines.
+        source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource()
+        source.file = lines[current + 1]
+        source.function = lines[current + 2]
+        source.line = int(lines[current + 3])
+        unique_id = lines[current + 5]
+
+        new_metadata = {'function_type': lines[current + 4],
+                        'extra_id': lines[current + 6],
+                        'unique_id_hash': _ComputeStringHash(unique_id)}
+        # Extract serialized proto.
+        current += 7
+        annotation_text = ""
+
+        while current < len(lines):
+          current += 1

[msramek, 2017/05/29 21:03:37]

optional: Consider moving this current += 1 at the end of the body of the large
while statement? I think it's a bit more readable when we just make this +1 step
to the next row at the end rather than always having to do [-1]. Possibly also
add a comment why is that += 1 there.

[Ramin Halavati, 2017/05/30 04:40:36]

Done.

+          if lines[current - 1] == "==== ANNOTATION ENDS ====":
+            break
+          else:
+            annotation_text += lines[current - 1]
+        else:
+          raise Exception(
+            "Error at line %i, expected annotation end tag." % current)
+

[msramek, 2017/05/29 21:03:37]

Should we handle an empty |annotation_text| here?

[Ramin Halavati, 2017/05/30 04:40:36]

It would be checked later when proto is deserialized. I will add it to unittest
so that it ensures that it's tested somewhere.

+        # Process unittests and undefined tags.
+        if unique_id in ("test", "test_partial"):
+          continue
+        if unique_id in ("undefined", "missing"):
+          errors.append("Annotation is not defined for file '%s', line %i." %
+              (source.file, source.line))
+          continue
+
+        # Decode serialized proto.
+        annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation()
+        try:
+          text_format.Parse(annotation_text, annotation_proto)
+        except Exception as error:
+          errors.append("Annotation in file '%s', line %i, has error: %s" %

[msramek, 2017/05/29 21:03:37]

nit: an error

[Ramin Halavati, 2017/05/30 04:40:36]

Done.

+              (source.file, source.line, error))
+
+        # Add new proto.
+        annotation_proto.unique_id = unique_id
+        annotation_proto.source.CopyFrom(source)
+        annotations.network_traffic_annotation.add().CopyFrom(annotation_proto)
+        metadata.append(new_metadata)
+      elif lines[current] == "==== NEW CALL ====":
+        # Ignore calls for now.
+        while current < len(lines):
+          current += 1
+          if lines[current - 1] == "==== CALL ENDS ====":
+            break
         else:
-          annotation_text += lines[current - 1]
-      else:
+          raise Exception(
+              "Error at line %i, expected call end tag." % current)
+      else: # The line is neither new annotation nor new call.
         raise Exception(
-          "Error at line %i, expected annotation end tag." % current)
-
-      # Process unittests and undefined tags.
-      if unique_id in ("test", "test_partial"):
-        continue
-      if unique_id in ("undefined", "missing"):
-        errors.append("Annotation is not defined for file '%s', line %i." %
-            (source.file, source.line))
-        continue
-
-      # Decode serialized proto.
-      annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation()
-      try:
-        text_format.Parse(annotation_text, annotation_proto)
-      except Exception as error:
-        errors.append("Annotation in file '%s', line %i, has error: %s" %
-            (source.file, source.line, error))
-
-      # Add new proto.
-      annotation_proto.unique_id = unique_id
-      annotation_proto.source.CopyFrom(source)
-      annotations.network_traffic_annotation.add().CopyFrom(annotation_proto)
-      metadata.append(new_metadata)
+            "Error at line %i, expected starting new annotation or call." %
+            current)
 
   except Exception as error:
     errors.append(str(error))