Tool added to extract network traffic annotations.

tools/clang/traffic_annotation_extractor/traffic_annotation_extractor.cpp

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This clang tool finds all instances of net::DefineNetworkTrafficAnnotation in
+// given source code, extracts the location info and content of annotation tags
+// (unique id and annotation text), and stores them in separate text files
+// (per instance) in the given output directory. Please refer to README.md for
+// build and usage instructions.
+
+#include <stdio.h>
+#include <fstream>
+#include <memory>
+
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace clang::ast_matchers;
+
+namespace {
+
+// An instance of a call to the net::DefineNetworkTrafficAnnotation function.
+struct NetworkAnnotationInstance {
+  // Information about where the call has happened.
+  struct Location {
+    std::string file_path;
+    int line_number = -1;
+
+    // Name of the function calling net::DefineNetworkTrafficAnnotation. E.g.,
+    // in the following code, |function_name| will be 'foo':
+    // void foo() { NetworkTrafficAnnotationTag bar =
+    //                  net::DefineNetworkTrafficAnnotation(...); }
+    std::string function_name;
+  };
+
+  // Annotation content. These are the parameters of the call to
+  // net::DefineNetworkTrafficAnnotation. The unique_id is an identifier for the
+  // annotation that has to be unique across the entire code base. The |text|
+  // stores a raw string with the annotation that should be extracted.
+  struct Annotation {
+    std::string unique_id;
+    std::string text;
+  };
+
+  Location location;
+  Annotation annotation;
+};
+
+using Collector = std::vector<NetworkAnnotationInstance>;
+
+// Returns the function that contains the given token. For example, if the token
+// is variable x in the code "void foo() { int x; ... }", it returns "foo".
+llvm::StringRef GetDeclarationCoveringFunction(const clang::Decl* token,

[dcheng, 2017/04/11 07:26:42]

It's usually nicer to express things as matchers when possible.

So we could do:

auto is_define_network_traffic_annotation = hasDeclaration(...);
match_finder.addMatcher(callExpr(
        is_define_network_traffic_annotation,
        hasAncestor(functionDecl().bind("functionContext")),
    &in_function_callback);
match_finder.addMatcher(callExpr(
        is_define_network_traffic_annotation,
        unless(hasAncestor(functionDecl())),
    &not_in_function_callback);

WDYT? We could actually just use the same callback for both,
and gate it on whether or not functionContext is bound.

[Ramin Halavati, 2017/04/11 09:15:38]

WOW! That was wonderful.

+                                               clang::ASTContext* context);
+
+// Returns the function that contains the given token. For example, if the token
+// is the call to function bar() in the code "void foo() { bar(); }", it returns
+// "foo".
+llvm::StringRef GetStatementCoveringFunction(const clang::Stmt* token,
+                                             clang::ASTContext* context) {
+  // Get the parent of |token| and return its covering function.
+  clang::ASTContext::DynTypedNodeList parents = context->getParents(*token);
+
+  // If parent is found, extract its name recursively.
+  if (parents.size()) {
+    if (const clang::Stmt* s = parents[0].get<clang::Stmt>())
+      return GetStatementCoveringFunction(s, context);
+    else if (const clang::Decl* d = parents[0].get<clang::Decl>())
+      return GetDeclarationCoveringFunction(d, context);
+  }
+
+  return llvm::StringRef("Unknown");
+}
+
+// Returns the function that contains the given token. For example, if the token
+// is variable x in the code "void foo() { int x; ... }", it returns "foo".
+llvm::StringRef GetDeclarationCoveringFunction(const clang::Decl* token,
+                                               clang::ASTContext* context) {
+  // If |token| is a function declaration, return its name.
+  if (auto f = clang::dyn_cast<clang::FunctionDecl>(token))
+    return f->getQualifiedNameAsString();
+
+  // As |token| is not a function declaration, get its parent and return its
+  // covering function.
+  clang::ASTContext::DynTypedNodeList parents = context->getParents(*token);
+
+  // If parent is found, extract its name recursively.
+  if (parents.size()) {
+    if (const clang::Stmt* s = parents[0].get<clang::Stmt>())
+      return GetStatementCoveringFunction(s, context);
+    else if (const clang::Decl* d = parents[0].get<clang::Decl>())
+      return GetDeclarationCoveringFunction(d, context);
+  }
+
+  return llvm::StringRef("Unknown");

[dcheng, 2017/04/11 07:26:42]

Btw, no need to explicit construct StringRef here.

[Ramin Halavati, 2017/04/11 09:15:38]

Done.

+}
+
+// This class implements the call back functions for AST Matchers. The matchers
+// are defined in RunMatchers function. When a pattern is found there,
+// the run function in this class is called back with information on the matched
+// location and description of the matched pattern.
+class NetworkAnnotationTagCallback : public MatchFinder::MatchCallback {
+ public:
+  explicit NetworkAnnotationTagCallback(Collector* collector)
+      : collector_(collector) {}
+  ~NetworkAnnotationTagCallback() override = default;
+
+  // Is called on any pattern found by ASTMathers that are defined in RunMathers
+  // function.
+  virtual void run(const MatchFinder::MatchResult& result) override {
+    const clang::CallExpr* call_expr =
+        result.Nodes.getNodeAs<clang::CallExpr>("definition_function");
+    const clang::StringLiteral* unique_id =
+        result.Nodes.getNodeAs<clang::StringLiteral>("unique_id");
+    const clang::StringLiteral* annotation_text =
+        result.Nodes.getNodeAs<clang::StringLiteral>("annotation_text");
+
+    if (call_expr && unique_id && annotation_text) {

[dcheng, 2017/04/11 07:26:42]

Let's assert since we should expect all these to be non-null.

[Ramin Halavati, 2017/04/11 09:15:38]

Done.

+      NetworkAnnotationInstance instance;
+      instance.annotation.unique_id = unique_id->getString();
+      instance.annotation.text = annotation_text->getString();
+
+      // Get annotation location.
+      clang::SourceLocation source_location = call_expr->getLocStart();
+      instance.location.file_path =
+          result.SourceManager->getFilename(source_location);
+      instance.location.line_number =
+          result.SourceManager->getSpellingLineNumber(source_location);
+      instance.location.function_name = GetStatementCoveringFunction(
+          clang::dyn_cast<clang::Stmt>(call_expr), result.Context);
+
+      // If DefineNetworkTrafficAnnotation is used in form of a macro, an empty
+      // file_path is returned. Traversing to its parent node in this case
+      // will result in the correct value.
+      if (!instance.location.file_path.length()) {

[dcheng, 2017/04/11 07:26:42]

Can we do something with getImmediateMacroCallerLoc() to jump out of scratch
space? For example, see
https://cs.chromium.org/chromium/src/tools/clang/rewrite_to_chrome_style/Rewr...

[Ramin Halavati, 2017/04/11 09:15:38]

Done.

+        instance.location.file_path = "unknown_file_path";
+        clang::ASTContext::DynTypedNodeList parents =
+            result.Context->getParents(
+                *clang::dyn_cast<clang::Stmt>(call_expr));
+        if (parents.size()) {
+          if (const clang::Stmt* s = parents[0].get<clang::Stmt>()) {
+            source_location = s->getLocStart();
+            instance.location.file_path =
+                result.SourceManager->getFilename(source_location);
+            instance.location.line_number =
+                result.SourceManager->getSpellingLineNumber(source_location);
+          }
+        }
+      }
+
+      // Remove leading "../"s from file path.
+      while (instance.location.file_path.length() > 3 &&
+             instance.location.file_path.substr(0, 3) == "../") {

[dcheng, 2017/04/11 07:26:42]

I would be cautious of doing this, as Windows will sometimes have \ in the name.

One easy hack to work around this is to just replace all \\ in the name with /
-- we can't allow filenames like that in the repo, since it won't work on
Windows.

[Ramin Halavati, 2017/04/11 09:15:38]

I did the replacement, but I am not sure if I got the last sentence correctly.
The reason I am doing this the file paths are passed to us starting with several
"../"s and I want to remove it from temporary file names.

+        instance.location.file_path = instance.location.file_path.substr(
+            3, instance.location.file_path.length() - 3);
+      }
+
+      collector_->push_back(instance);
+    }
+  }
+
+ private:
+  Collector* collector_;
+};
+
+// Sets up an ASTMatcher and runs clang tool to populate collector. Returns the
+// result of running the clang tool.
+int RunMatchers(clang::tooling::ClangTool* clang_tool, Collector* collector) {
+  NetworkAnnotationTagCallback call_back(collector);
+  MatchFinder match_finder;
+
+  // Set up a pattern to find functions that are named
+  // [net::]DefineNetworkTrafficAnnotation and have 2 arguments of string
+  // literal type.
+  match_finder.addMatcher(
+      callExpr(hasDeclaration(functionDecl(
+                   anyOf(hasName("DefineNetworkTrafficAnnotation"),
+                         hasName("net::DefineNetworkTrafficAnnotation")))),
+               hasArgument(0, stringLiteral().bind("unique_id")),
+               hasArgument(1, stringLiteral().bind("annotation_text")))
+          .bind("definition_function"),
+      &call_back);
+  std::unique_ptr<clang::tooling::FrontendActionFactory> frontend_factory =
+      clang::tooling::newFrontendActionFactory(&match_finder);
+  return clang_tool->run(frontend_factory.get());
+}
+
+}  // namespace
+
+static llvm::cl::OptionCategory ToolCategory(
+    "traffic_annotation_extractor: Extract traffic annotation texts");
+static llvm::cl::extrahelp CommonHelp(
+    clang::tooling::CommonOptionsParser::HelpMessage);
+static llvm::cl::extrahelp MoreHelp(
+    "\n  -output-directory=<string>    - The directory in which the output "
+    "files are written.\n");
+static llvm::cl::opt<std::string> OutputDirectory(
+    "output-directory",
+    llvm::cl::desc("The directory in which output files are written.\n"));
+
+int main(int argc, const char* argv[]) {
+  clang::tooling::CommonOptionsParser options(argc, argv, ToolCategory);
+  clang::tooling::ClangTool tool(options.getCompilations(),
+                                 options.getSourcePathList());
+  Collector collector;
+
+  int result = RunMatchers(&tool, &collector);
+
+  if (result != 0)
+    return result;
+
+  // For each call to "DefineNetworkTrafficAnnotation", write annotation text

[dcheng, 2017/04/11 07:26:42]

Btw, I'm curious if we still need to do this: for the Blink rename, we split the
tool up into several stages. Before, run_tool.py would do everything, including
applying edits.

Now  run_tool.py prints the stdout of tools to stdout, so the previous pipeline
now looks like:

run_tool.py > output
cat output extract_edits.py| apply_edits.py

I think for network annotations, we can just substitute something else other
than extract_edits.py | apply_edits.py. That should make the logic simpler
overall for traffic_annotation_auditor.cc

[Ramin Halavati, 2017/04/11 09:15:38]

Thanks, Done!

+  // and relevant meta data into a separate file. The filename is uniquely
+  // generated using the file path and filename of the code including the call
+  // and it's line number.

[dcheng, 2017/04/11 07:26:42]

Nit: it's -> its

[Ramin Halavati, 2017/04/11 09:15:38]

Done.

+  for (NetworkAnnotationInstance& call : collector) {
+    std::string s = call.location.file_path;
+    std::replace(s.begin(), s.end(), '/', '_');
+    std::replace(s.begin(), s.end(), '.', '_');
+    std::string file_path = OutputDirectory.getValue() + "/" + s + "(" +
+                            std::to_string(call.location.line_number) + ").txt";
+
+    std::ofstream output_file(file_path);
+    if (output_file.is_open()) {
+      output_file << call.location.file_path << "\n";
+      output_file << call.location.function_name << "\n";
+      output_file << call.location.line_number << "\n";
+      output_file << call.annotation.unique_id << "\n";
+      output_file << call.annotation.text << "\n";
+      output_file.close();
+    } else {
+      llvm::errs() << "Could not write to file: " << file_path << "\n";
+      return 1;
+    }
+  }
+
+  return 0;
+}