Tool added to extract network traffic annotations.

tools/traffic_annotation/auditor/traffic_annotation_auditor.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See README.md for information and build instructions.
+
+#include "base/command_line.h"
+#include "base/files/file_enumerator.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/files/scoped_temp_dir.h"
+#include "base/logging.h"
+#include "base/process/launch.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_split.h"
+#include "base/strings/stringprintf.h"
+#include "base/strings/utf_string_conversions.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "third_party/protobuf/src/google/protobuf/text_format.h"
+
+#include "tools/traffic_annotation/traffic_annotation.pb.h"
+
+namespace {
+
+// Holds an instance of network traffic annotation.
+struct AnnotationInstance {
+  AnnotationInstance() : state(STATE_OK) {}
+
+  // Protobuf of the annotation.
+  traffic_annotation::NetworkTrafficAnnotation proto;
+  // Unique id of the annotation.
+  std::string unique_id;
+  // State of this annotation.
+  enum State { STATE_OK, STATE_DUPLICATE, STATE_UNDEFINED } state;
+};
+
+// Runs clang tool, returns true if all steps are successful. Returns annotation
+// extractor clang tool's exit code in |clang_tool_exit_code|.
+bool RunClangTool(const base::FilePath& src_dir,
+                  const base::FilePath& build_dir,
+                  const base::FilePath& output_dir,
+                  const base::CommandLine::StringVector& path_filters,
+                  int* clang_tool_exit_code) {
+#ifdef _WIN32
+  // Run generate_win_compdb.py on Windows to repair compdb before running clang
+  // tool.
+  base::CommandLine::StringVector argv;
+  argv.push_back(L"python");
+  argv.push_back(base::ASCIIToUTF16(
+      src_dir.AppendASCII("tools/clang/scripts/generate_win_compdb.py")
+          .MaybeAsASCII()));
+  argv.push_back(base::ASCIIToUTF16(build_dir.MaybeAsASCII()));
+  int exit_code = -1;
+  base::Process process =
+      base::LaunchProcess(base::CommandLine(argv), base::LaunchOptions());
+  if (!process.IsValid() || !process.WaitForExit(&exit_code) || exit_code) {
+    LOG(INFO) << " Executing generate_win_compdb failed.\n";
+    return false;
+  }
+#endif
+
+  for (auto& path : path_filters) {
+    // Create commandline based on OS.
+    base::CommandLine::StringVector argv;
+#ifdef _WIN32
+    argv.push_back(L"python");
+    argv.push_back(base::ASCIIToUTF16(
+        src_dir.AppendASCII("tools/clang/scripts/run_tool.py").MaybeAsASCII()));

[battre, 2017/04/07 08:42:28]

Do you need to go through ASCII here and in the following? Doesn't that break if
src_dir contains non-ascii characters?

Can't you work with .value()?

[Ramin Halavati, 2017/04/07 11:33:33]

I couldn't make it work any simpler way. I will discuss it with Windows
developers group to see if I can force windows compile to be ascii similar to
linux compile, this way I will get read of all #ifdef _WIN32 parts.

+    argv.push_back(L"traffic_annotation_extractor");
+    argv.push_back(base::ASCIIToUTF16(build_dir.MaybeAsASCII()));
+    argv.push_back(path);
+    argv.push_back(base::ASCIIToUTF16(
+        base::StringPrintf("--tool-args=-output-directory=%s",
+                           output_dir.MaybeAsASCII().c_str())));
+#else
+    argv.push_back(
+        src_dir.AppendASCII("tools/clang/scripts/run_tool.py").MaybeAsASCII());
+    argv.push_back("--generate-compdb");
+    argv.push_back("traffic_annotation_extractor");
+    argv.push_back(build_dir.MaybeAsASCII());
+    argv.push_back(path);
+    argv.push_back(base::StringPrintf("--tool-args=-output-directory=%s",
+                                      output_dir.MaybeAsASCII().c_str()));
+#endif
+
+    base::Process process =
+        base::LaunchProcess(base::CommandLine(argv), base::LaunchOptions());
+    if (!process.IsValid() || !process.WaitForExit(clang_tool_exit_code)) {
+      LOG(ERROR) << "Executing clang tool failed.\n";
+      return false;
+    }
+  }
+  return true;
+}
+
+// Reads an extracted annotation file and returns it in the output variables.
+// The file starts with four |header_lines| with the following meaning:
+//   0- File path.
+//   1- Name of the function including this position.
+//   2- Line number.
+//   3- Unique id of annotation.
+// The rest of the file is the protobuf text (|msg_text|).
+bool ReadFile(const base::FilePath& file_path,
+              std::vector<std::string>* header_lines,
+              std::string* msg_text) {
+  std::string file_content;
+  if (!base::ReadFileToString(file_path, &file_content))
+    return false;
+
+  header_lines->clear();
+  msg_text->clear();
+
+  std::vector<std::string> tokens = base::SplitString(
+      file_content, "\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_ALL);
+
+  // If enough data is extracted, populate outputs, otherwise leave them blank.
+  if (tokens.size() > 4) {
+    for (int i = 0; i < 4; i++)
+      header_lines->push_back(tokens[i]);
+    for (size_t i = 4; i < tokens.size(); i++)
+      *msg_text += tokens[i] + "\n";
+  }
+
+  return true;
+}
+
+// Reads all extracted txt files from given input folder and populates instances
+// and errors. Errors include not finding the file, incorrect content, or error
+// passed from clang tool.
+void ReadExtractedFiles(const base::FilePath& folder_name,
+                        std::vector<AnnotationInstance>* instances,
+                        std::vector<std::string>* errors) {
+#ifdef _WIN32
+  base::FileEnumerator file_iter(folder_name, false,
+                                 base::FileEnumerator::FILES, L"*.txt");
+#else
+  base::FileEnumerator file_iter(folder_name, false,
+                                 base::FileEnumerator::FILES, "*.txt");
+#endif
+  while (!file_iter.Next().empty()) {
+    std::string file_name = file_iter.GetInfo().GetName().AsUTF8Unsafe();
+    LOG(INFO) << "Reading " << file_name.c_str() << "...\n";
+
+    std::vector<std::string> header_lines;
+    std::string msg_text;
+    if (!ReadFile(folder_name.Append(file_iter.GetInfo().GetName()),
+                  &header_lines, &msg_text)) {
+      errors->push_back(
+          base::StringPrintf("Could not open file '%s'\n.", file_name.c_str()));
+      continue;
+    }
+    if (header_lines.size() < 4) {
+      errors->push_back(base::StringPrintf(
+          "Header lines are not complete for file '%s'\n.", file_name.c_str()));
+      continue;
+    }
+
+    AnnotationInstance new_instance;
+    if (header_lines[3] == "\"Undefined\"") {
+      new_instance.state = AnnotationInstance::STATE_UNDEFINED;
+    } else if (!google::protobuf::TextFormat::ParseFromString(
+                   msg_text, (google::protobuf::Message*)&new_instance)) {
+      errors->push_back(base::StringPrintf(
+          "Could not parse protobuf for file '%s'\n.", file_name.c_str()));
+      continue;
+    }
+
+    // Add header data to new instance.
+    traffic_annotation::NetworkTrafficAnnotation_TrafficSource* src =
+        new_instance.proto.mutable_source();
+    src->set_file(header_lines[0]);
+    src->set_function(header_lines[1]);
+    int line;
+    base::StringToInt(header_lines[2], &line);
+    src->set_line(line);
+    new_instance.unique_id = header_lines[3];
+    instances->push_back(new_instance);
+  }
+  LOG(INFO) << instances->size() << " annotation instance(s) read.\n";
+}
+
+// Checks to see if unique ids are really unique and marks the ones which are
+// not.
+void MarkRepeatedUniqueIds(std::vector<AnnotationInstance>* instances) {
+  std::map<std::string, AnnotationInstance*> unique_ids;
+  for (auto& instance : *instances) {
+    if (instance.state == AnnotationInstance::STATE_OK) {
+      auto match = unique_ids.find(instance.unique_id);
+      if (match != unique_ids.end())

[battre, 2017/04/07 08:42:28]

please use {} if lines are wrapped also ein the else branch.

[Ramin Halavati, 2017/04/07 11:33:33]

Done.

+        instance.state = match->second->state =
+            AnnotationInstance::STATE_DUPLICATE;
+      else
+        unique_ids.insert(
+            std::make_pair(std::string(instance.unique_id), &instance));
+    }
+  }
+}
+
+// Writes summary of annotation instances to a file. Returns true if successful.
+bool WriteSummaryFile(int clang_tool_exit_code,
+                      const std::vector<AnnotationInstance>& instances,
+                      const std::vector<std::string>& errors,
+                      const base::FilePath& file_path) {
+  std::string report = "";
+
+  if (errors.size() || clang_tool_exit_code) {
+    report = "[Errors]\n";
+
+    if (clang_tool_exit_code)
+      report += base::StringPrintf("Clang tool returned error: %i\n",
+                                   clang_tool_exit_code);
+
+    for (const auto& error : errors)
+      report += error + "\n";
+  }
+
+  report += "[Annotations]\n";
+
+  for (const auto& instance : instances) {
+    report +=
+        "------------------------------------------------------------"
+        "--------------------\n";
+    report += base::StringPrintf("Unique ID: %s\n", instance.unique_id.c_str());
+    if (instance.state == AnnotationInstance::STATE_UNDEFINED)
+      report += base::StringPrintf("WARNING: Undefined annotation.\n");
+    else if (instance.state == AnnotationInstance::STATE_DUPLICATE)
+      report += base::StringPrintf("WARNING: Duplicate unique id.\n");
+    std::string temp;
+    google::protobuf::TextFormat::PrintToString(instance.proto, &temp);
+    report += base::StringPrintf("Content:\n%s\n", temp.c_str());
+  }
+
+  if (base::WriteFile(file_path, report.c_str(), report.length()) == -1) {
+    LOG(INFO) << " Could not create output file: "
+              << file_path.MaybeAsASCII().c_str() << ".\n";
+    return false;
+  }
+
+  LOG(INFO) << "Output file " << file_path.MaybeAsASCII().c_str()
+            << " written for " << instances.size() << " instances.\n";
+  return true;
+}
+
+}  // namespace
+
+#ifdef _WIN32
+int wmain(int argc, wchar_t* argv[]) {
+#else
+int main(int argc, char* argv[]) {
+#endif
+  // Parse switches.
+  base::CommandLine command_line = base::CommandLine(argc, argv);
+  if (command_line.HasSwitch("help") || command_line.HasSwitch("h") ||
+      argc == 1) {
+    LOG(INFO)
+        << "Usage: traffic_annotation_auditor [OPTION]... [path_filter]...\n"
+           "Extracts network traffic annotations from source files. If path "
+           "filter(s) are specified, only those directories of the source "
+           "will be analyzed.\n"
+           "Options:\n"
+           "  -h, --help              Shows help.\n"
+           "  --build-dir             Path to the build directory from which "
+           "the\n"
+           "                            annotations will be extracted.\n"
+           "  --extractor-output-dir  Path to the directory that extracted\n"
+           "                            partial files will be written to. "
+           "Will\n"
+           "                            be automatically generated and "
+           "deleted\n"
+           "                            if not specified.\n"
+           "  --extracted-input-dir   Path to a directory where extracted\n"
+           "                            partial annotations are already "
+           "stored.\n"
+           "                            If specified, build directory will be\n"
+           "                            ignored.\n"
+           "  --summary-file          Path to an output file with summary of\n"
+           "                            extracted annotations.\n"
+           "Example:\n"
+           "  traffic_annotation_auditor --build-dir=out/Debug\n"
+           "    --summary-file=report.txt\n";
+    return 1;
+  }
+  base::FilePath extracted_files_dir =
+      command_line.GetSwitchValuePath("extracted-input-dir");
+  base::FilePath build_dir = command_line.GetSwitchValuePath("build-dir");
+  base::FilePath extractor_output_dir =
+      command_line.GetSwitchValuePath("extractor-output-dir");
+  base::FilePath summary_file = command_line.GetSwitchValuePath("summary-file");
+  base::ScopedTempDir temp_dir;
+
+  if (summary_file.empty())
+    LOG(ERROR) << "WARNING: Output file not specified.\n";
+
+  int clang_tool_exit_code = 0;
+
+  // Extract annotations.
+  if (extracted_files_dir.empty()) {
+    // Get build directory, if it is empty issue an error.
+    if (build_dir.empty()) {
+      LOG(ERROR)
+          << "You must either specify build directory to run clang tool and "
+             "extract annotations, or specify extracted files's input "

[battre, 2017/04/07 08:42:28]

You must either specify the build directory to run clang tool and extract
annotations, or specify specify the input directory where extracted annotation
files already exist.

[Ramin Halavati, 2017/04/07 11:33:33]

Done.

+             "directory where already extracted files exist.\n";
+      return 1;
+    }
+    // If output directory is not provided, create a temporary one.
+    if (extractor_output_dir.empty()) {
+      if (!temp_dir.CreateUniqueTempDirUnderPath(build_dir)) {
+        LOG(ERROR) << "Could not create temporary directory in "
+                   << build_dir.MaybeAsASCII().c_str() << ".\n";
+        return 1;
+      }
+      extractor_output_dir = temp_dir.GetPath();
+    } else {
+      // Ensure given directory is empty.
+      if (!base::FileEnumerator(extractor_output_dir, false,
+                                base::FileEnumerator::FILES)
+               .Next()
+               .empty()) {
+        LOG(ERROR) << "Output directory "
+                   << extractor_output_dir.MaybeAsASCII().c_str()
+                   << "should be empty .\n";
+        return 1;
+      }
+    }
+
+    // Get path filters, if none is provided, just add all.
+    base::CommandLine::StringVector path_filters = command_line.GetArgs();
+    if (!path_filters.size()) {
+      base::FilePath temp;
+      path_filters.push_back(temp.AppendASCII("./").value().c_str());
+    }
+
+    // Eexcutable is usually in out/[Build Dir], so the path to source is
+    // extracted by moving two directories up.
+    if (!RunClangTool(command_line.GetProgram().DirName().AppendASCII("../.."),
+                      build_dir, extractor_output_dir, path_filters,
+                      &clang_tool_exit_code)) {
+      return 1;
+    }
+
+    extracted_files_dir = extractor_output_dir;
+  }
+
+  // Read all extracted files.
+  std::vector<AnnotationInstance> instances;
+  std::vector<std::string> errors;
+  ReadExtractedFiles(extracted_files_dir, &instances, &errors);
+
+  if (instances.empty()) {
+    LOG(ERROR) << "Could not read any file.\n";
+    return 1;
+  } else {
+    MarkRepeatedUniqueIds(&instances);
+  }
+
+  // Create Summary file if requested.
+  if (!summary_file.empty() &&
+      !WriteSummaryFile(clang_tool_exit_code, instances, errors, summary_file))
+    return 1;
+
+  return 0;
+}