Tool added to extract network traffic annotations.

tools/traffic_annotation/traffic_annotation_auditor.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See README.txt for information and build instructions.

[battre, 2016/10/26 14:29:20]

Please add this.

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <map>
+#include <string>
+
+#include "base/command_line.h"
+#include "base/files/file_enumerator.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/strings/stringprintf.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "third_party/protobuf/src/google/protobuf/text_format.h"
+
+#include "tools/traffic_annotation/traffic_annotation.pb.h"
+
+// Holds an instance of network traffic annotation
+struct Instance {
+  Instance() {

[battre, 2016/10/26 14:29:19]

use initializer list?

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+    state = ok;
+    hash_code = 0;
+  }
+  traffic_annotation::NetworkTrafficAnnotation proto;
+  std::string unique_id;
+  std::string callee;
+  uint hash_code;

[battre, 2016/10/26 14:29:20]

I think that C++ makes no definition of the size of an uint. This should be a
fixed size integer (also in the other CL)

[Ramin Halavati, 2016/10/27 09:40:10]

In the other CL "unsigned" type is returned by several clang function, hence I
used it. I replaced uint with unsigned int, which is used by our hash generation
function. Is it OK?

+  enum { ok, duplicate, undefined } state;

[battre, 2016/10/26 14:29:19]

See Style guide on the naming of enums.

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+};
+
+// Runs clang tool, returns true if all OK.
+bool RunClangTool(const base::FilePath& src_dir,
+                  const base::FilePath& build_dir,
+                  const base::FilePath& output_dir,
+                  const base::CommandLine::StringVector& path_filters) {
+  for (auto& path : path_filters) {
+    int result = system(
+        base::StringPrintf(
+            "%s traffic_annotation_extractor"
+            " --generate-compdb %s %s --tool-params 'output_dir=%s'",
+            src_dir.Append("tools/clang/scripts/run_tool.py").value().c_str(),
+            build_dir.value().c_str(), path.c_str(), output_dir.value().c_str())
+            .c_str());
+    if (result) {
+      printf(" Executing clang tool returned code: %i.\n", result);
+      return false;
+    }
+  }
+  return true;
+}
+
+// Reads an extrated annotation file. The first 6 lines of the file include:
+//   0- File path and file name.
+//   1- Name of the function that is called using annotation.
+//   2- Line number.
+//   3- Name of the function including this position.
+//   4- Error text associated with this annotation.
+//   5- Unique id of annotation.
+// The rest of the file is the protobuf text.
+bool ReadFile(const base::FilePath& file_path,
+              std::vector<std::string>* header_lines,
+              std::string* msg_text) {
+  std::ifstream input_file(file_path.value());
+  if (!input_file.is_open())
+    return false;
+

[battre, 2016/10/26 14:29:19]

msg_text->clear();

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+  while (!input_file.eof()) {
+    std::string temp;

[battre, 2016/10/26 14:29:20]

std::string line;?

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+    getline(input_file, temp);
+    if (header_lines->size() < 6)
+      header_lines->push_back(temp);
+    else
+      *msg_text = *msg_text + "\n" + temp;
+  }
+  input_file.close();
+  return true;
+}
+
+// Reads all extracted txt files form given input folder and populates instances
+// and errors.
+void ReadExtractedFiles(const base::FilePath& folder_name,
+                        std::vector<Instance>* instances) {
+  base::FileEnumerator file_iter(folder_name, false,
+                                 base::FileEnumerator::FILES, "*.txt");
+  while (!file_iter.Next().empty()) {
+    std::string file_name = file_iter.GetInfo().GetName().AsUTF8Unsafe();
+    printf("\tReading %s...\n", file_name.c_str());
+
+    std::vector<std::string> header_lines;
+    std::string msg_text;
+    if (!ReadFile(folder_name.Append(file_iter.GetInfo().GetName()),
+                  &header_lines, &msg_text)) {
+      fprintf(stderr, "Could not open file '%s'.", file_name.c_str());
+      continue;
+    }
+    if (header_lines.size() < 6) {
+      fprintf(stderr, "Header lines are not complete for file '%s'.",
+              file_name.c_str());
+      continue;
+    }
+    if (!header_lines[4].empty()) {
+      fprintf(stderr, "Error passed from extractor for file '%s': %s.",
+              file_name.c_str(), header_lines[4].c_str());
+      continue;
+    }
+
+    Instance new_instance;
+    if (header_lines[5] == "\"Undefined\"") {
+      new_instance.state = Instance::undefined;
+    } else if (!google::protobuf::TextFormat::ParseFromString(
+                   msg_text, (google::protobuf::Message*)&new_instance)) {
+      fprintf(stderr, "Could not parse protobuf for file '%s'.",
+              file_name.c_str());
+      continue;
+    }
+
+    // Add header data to new instance.
+    traffic_annotation::NetworkTrafficAnnotation_TrafficSource* src =
+        new_instance.proto.mutable_source();
+    src->set_file(header_lines[0]);
+    src->set_function(header_lines[1]);
+    src->set_line(atoi(header_lines[2].c_str()));
+    new_instance.callee = header_lines[3];
+    new_instance.unique_id = header_lines[5];
+    new_instance.hash_code =
+        COMPUTE_STRING_HASH(new_instance.unique_id.c_str());
+    instances->push_back(new_instance);
+  }
+  printf("%i annotation instance(s) read.\n", (int)instances->size());
+}
+
+// Checks to see if two unique ids have similar hash codes.

[battre, 2016/10/26 14:29:20]

This function mutates the instances. That should be described and reflected in
the function.

MarkHashCollisions()?

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+void CheckDuplicateHashCodes(std::vector<Instance>* instances) {
+  std::map<uint, Instance*> hashed_ids;
+  for (auto& instance : *instances)

[battre, 2016/10/26 14:29:20]

{}

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+    if (instance.state == Instance::ok) {
+      auto p = hashed_ids.find(instance.hash_code);

[battre, 2016/10/26 14:29:20]

what is p? --> Too short variable name. Maybe "iter"?

[Ramin Halavati, 2016/10/27 09:40:10]

Renamed to match.

+      if (p != hashed_ids.end())
+        instance.state = p->second->state = Instance::duplicate;
+      else
+        hashed_ids.insert(std::make_pair(instance.hash_code, &instance));
+    }
+}
+
+// Writes summary of annotation instances to a file. Returns true if successful.
+bool WriteSummaryFile(const std::vector<Instance>& instances,
+                      const std::string& file_name) {
+  FILE* output_file = fopen(file_name.c_str(), "wt");
+  if (!output_file) {
+    printf(" Could not create output file: %s", file_name.c_str());
+    return false;
+  }
+  for (auto& instance : instances) {

[battre, 2016/10/26 14:29:19]

const

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+    fprintf(output_file, "Unique ID: %s\tHash code:%u\n",
+            instance.unique_id.c_str(), instance.hash_code);
+    if (instance.state == Instance::undefined)
+      fprintf(output_file, "WARNING: Undefined annotation.\n");
+    else if (instance.state == Instance::duplicate)
+      fprintf(output_file, "WARNING: Duplicate hash code for unique id.\n");
+    fprintf(output_file, "Calling Function: %s\n", instance.callee.c_str());
+    std::string temp;
+    google::protobuf::TextFormat::PrintToString(instance.proto, &temp);
+    fprintf(output_file, "Content:\n%s\n", temp.c_str());
+    fprintf(output_file, "------------------------------------------------\n");
+  }
+  fclose(output_file);
+  printf("Output file %s written for %i instances.\n", file_name.c_str(),
+         (int)instances.size());
+  return true;
+}
+
+int main(int argc, char* argv[]) {
+  // Parse switches.
+  base::CommandLine command_line = base::CommandLine(argc, argv);
+  if (command_line.HasSwitch("help") || command_line.HasSwitch("h") ||
+      argc == 1) {
+    printf(
+        "Usage: traffic_annotation_auditor [OPTION]... [path_filter]...\n"
+        "Extracts network traffic annotations from source files. If path "
+        "filter(s) are specified, only those directories of the source "
+        "will be analyzed.\n"
+        "Options:\n"
+        "  -h, --help              Shows this.\n"
+        "  --build_dir             Path to the build directory from which the\n"
+        "                            annotations will be extracted.\n"
+        "  --extractor_output_dir  Path to the directory that extracted\n"
+        "                            partial files will be written to. Will\n"
+        "                            be automatically generated and deleted\n"
+        "                            if not specified.\n"
+        "  --extracted_input_dir   Path to a directory where extracted\n"
+        "                            partial annotations are already stored.\n"
+        "                            If specified, build directory will be\n"
+        "                            ignored.\n"
+        "  --summary_file          Path to an ouput file with summary of\n"
+        "                            extracted annotations.\n"
+        "Example:\n"
+        "  traffic_annotation_auditor --build_dir=out/Debug\n"
+        "    --summery_file=report.txt\n");
+    return 1;
+  }
+  base::FilePath extracted_files_dir =
+      command_line.GetSwitchValuePath("extracted_input_dir");
+  base::FilePath build_dir = command_line.GetSwitchValuePath("build_dir");
+  base::FilePath extractor_output_dir =
+      command_line.GetSwitchValuePath("extractor_output_dir");
+  base::FilePath summary_file = command_line.GetSwitchValuePath("summary_file");
+
+  // Extract annotations.
+  if (extracted_files_dir.empty()) {
+    // Get build directory, if it is empty issue an error.
+    if (build_dir.empty()) {
+      fprintf(
+          stderr,
+          "You must either specify build directory to run clang "
+          "tool and extract annotations, or specify extracted files's input "
+          "directory where already extracted files exist.\n");
+      return 1;
+    }
+    // Get output directory, if it is not provided, create a temporary one.
+    if (extracted_files_dir.empty()) {
+      if (!base::CreateTemporaryDirInDir(build_dir, "", &extracted_files_dir)) {
+        fprintf(stderr, "Could not create temporary directory in %s.",
+                build_dir.value().c_str());

[battre, 2016/10/26 14:29:20]

Unless you create a scoped temp directory, the directory does not get deleted on
shutdown.

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+        return 1;
+      }
+    } else {
+      // Ensure given directory is empty.
+      if (!base::FileEnumerator(extracted_files_dir, false,
+                                base::FileEnumerator::FILES, "*.txt")
+               .Next()
+               .empty()) {
+        fprintf(stderr, "Ouput directory (%s) should be empty.",
+                extracted_files_dir.value().c_str());
+        return 1;
+      }
+    }
+
+    // Get path filters, if none is provided, just add all.
+    base::CommandLine::StringVector path_filters = command_line.GetArgs();
+    if (!path_filters.size())
+      path_filters.push_back("./");
+
+    if (!RunClangTool(command_line.GetProgram().DirName().Append("../.."),
+                      build_dir, extracted_files_dir, path_filters))

[battre, 2016/10/26 14:29:19]

{}

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+      return 1;
+  }
+
+  // Read all extracted files.
+  std::vector<Instance> instances;
+  ReadExtractedFiles(extracted_files_dir, &instances);
+
+  if (instances.size()) {

[battre, 2016/10/26 14:29:20]

if (!instances.empty()) {

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+    CheckDuplicateHashCodes(&instances);
+  } else {
+    printf("Could not read any file.\n");
+    return 1;
+  }
+
+  // Create Summary file.
+  if (!summary_file.empty())

[battre, 2016/10/26 14:29:19]

{}

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+    if (!WriteSummaryFile(instances,
+                          command_line.GetSwitchValueASCII("summary_file")))

[battre, 2016/10/26 14:29:20]

{}

[Ramin Halavati, 2016/10/27 09:40:10]

Done.

+      return 1;
+
+  return 0;
+}