Tool added to extract network traffic annotations.

tools/clang/traffic_annotation_extractor/traffic_annotation_extractor.cpp

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+

[battre, 2016/10/27 11:49:06]

Do you want to make some comments here what this program is about?

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+#include <fstream>
+#include <memory>
+#include <stdio.h>
+
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+//--> TODO: #include "../../traffic_annotation/traffic_annotation.pb.h"
+
+using namespace clang::ast_matchers;
+
+namespace {
+
+// Structure to collect instances of network traffic annotation usages.
+struct Collector {
+  // An instance of network traffic annotation usage. This can be either
+  // a variable defined as NetworkTrafficAnnotationTag or a function that has
+  // a variable of this type as it's input parameter.
+  struct Instance {

[battre, 2016/10/27 11:49:06]

What do you think of moving this struct out of Collector?

[Ramin Halavati, 2016/10/27 12:51:20]

Done. Also renamed it to NetworkAnnotationInstance.

+    Instance()
+        : variable_reference(nullptr),
+          is_direct_call(false),
+          transitive_parameter(false) {}
+
+    // Information about where this annotation or call has happened.
+    struct Location {
+      Location() : line_number(-1) {}
+      std::string file_name;
+      int line_number;
+      // Name of the function including this instance.
+      std::string function_name;

[battre, 2016/10/27 11:49:06]

Can you add examples for this and object_name?

// Name of the function including this instance.
// e.g. ::Foo::my_function(void*)

(does it look like that?)

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+      // Name of the variable that contains annotation or the function called
+      // with annotation.
+      std::string object_name;
+    };
+
+    // Annotation content.
+    struct Annotation {
+      std::string unique_id;
+      std::string text;
+    };
+
+    Location location;
+    Annotation annotation;
+
+    // Possible error (empty if no error).
+    std::string error;
+    // A reference to the variable containing annotation. Null if not available.
+    const clang::NamedDecl* variable_reference;
+    // Flag stating that parameter is directly passed to annotate function here
+    // or it's through a variable.
+    bool is_direct_call;
+    // Flag stating that a variable is a parameter received by upper level
+    // function.
+    bool transitive_parameter;
+  };
+
+  std::vector<Instance> variable_definitions;
+  std::vector<Instance> calls;
+};
+
+// Returns the function that includes the given clang::Decl.
+std::string GetCoveringFunction(const clang::Decl* token,
+                                const MatchFinder::MatchResult& result);
+
+// Checks if a token matches a name, with or without net:: namespace.
+bool net_match(const std::string& token, const std::string& name) {
+  return token == name || token == (std::string("net::") + name);
+}
+
+// Returns the text of a given statement or subclass.

[battre, 2016/10/27 11:49:06]

Can you give an example of what this means?

[Ramin Halavati, 2016/10/27 12:51:20]

Done, but mostly updated the comment. A clang::Stmt can refer to any statement
of the AST and this function extracts its source code.

+std::string GetStmtText(const clang::Stmt* token,
+                        const clang::SourceManager& source_manager) {
+  clang::LangOptions lopt;
+  // Get text range

[battre, 2016/10/27 11:49:05]

.

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+  clang::SourceLocation start = token->getLocStart();
+  clang::SourceLocation end = token->getLocEnd();
+
+  // If it's a macro, go to definition.
+  if (start.isMacroID())
+    start = source_manager.getSpellingLoc(start);
+  if (end.isMacroID())
+    end = source_manager.getSpellingLoc(end);
+
+  // Get the real end of the token.
+  end = clang::Lexer::getLocForEndOfToken(end, 0, source_manager, lopt);
+
+  // Extract text.
+  std::string output(source_manager.getCharacterData(start),
+                     source_manager.getCharacterData(end) -
+                         source_manager.getCharacterData(start));

[battre, 2016/10/27 11:49:06]

std::string also has a constructor template <class InputIterator>
  string  (InputIterator first, InputIterator last);

I thought that you could simplify this further to 
std::string output(source_manager.getCharacterData(start),
                   source_manager.getCharacterData(end));

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+
+  // Raw string?
+  if (output == "R") {
+    if (auto* c1 = clang::dyn_cast<clang::ImplicitCastExpr>(token)) {
+      if (const clang::StringLiteral* c2 =
+              clang::dyn_cast<clang::StringLiteral>(
+                  c1->getSubExprAsWritten())) {
+        output = c2->getString();
+      }
+    }
+  }
+
+  return output;
+}
+
+// Extracts unique id and annotation text of a call to
+// "DefineNetworkTrafficAnnotation" function. Sets the error text if fails.
+void GetAnnotationText(const clang::CallExpr* call_expr,
+                       const clang::SourceManager& source_manager,
+                       Collector::Instance* instance) {
+  if (net_match(GetStmtText(call_expr->getCallee(), source_manager),
+                "DefineNetworkTrafficAnnotation") &&
+      call_expr->getNumArgs() == 2) {
+    instance->annotation.unique_id =
+        GetStmtText(call_expr->getArgs()[0], source_manager);
+    instance->annotation.text =
+        GetStmtText(call_expr->getArgs()[1], source_manager);
+    instance->error = "";
+  } else {
+    instance->annotation.unique_id = "";
+    instance->annotation.text = "";
+    instance->error = "Unexpected function.";
+  }
+}
+
+// Returns the function that includes the given clang::Stmt.

[battre, 2016/10/27 11:49:06]

Example? If token is a statement like "...", the function returns "...". If it
is a declaration like ...

I guess my problem is that I am not an expert with Clang tools but other
reviewers may suffer from that issue as well.

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+std::string GetCoveringFunction(const clang::Stmt* token,
+                                const MatchFinder::MatchResult& result) {
+  auto parents = result.Context->getParents(*token);

[battre, 2016/10/27 11:49:06]

Use DynTypedNodeList instead of auto here?

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+  // TODO: What exactly != 1 parent mean?
+  if (parents.size() == 1) {
+    if (const clang::Stmt* s = parents[0].get<clang::Stmt>())
+      return GetCoveringFunction(s, result);
+    else if (const clang::Decl* d = parents[0].get<clang::Decl>())
+      return GetCoveringFunction(d, result);
+  }
+  return "Unknown";
+}
+
+// Returns the function that includes the given clang::Decl.
+std::string GetCoveringFunction(const clang::Decl* token,
+                                const MatchFinder::MatchResult& result) {
+  if (auto f = clang::dyn_cast<clang::FunctionDecl>(token))
+    return f->getQualifiedNameAsString();
+
+  auto parents = result.Context->getParents(*token);
+  // TODO: What exactly != 1 parent mean?
+  if (parents.size() == 1) {
+    if (const clang::Stmt* s = parents[0].get<clang::Stmt>())
+      return GetCoveringFunction(s, result);
+    else if (const clang::Decl* d = parents[0].get<clang::Decl>())
+      return GetCoveringFunction(d, result);
+  }
+  return "Unknown";
+}
+
+// Finds file name and line number of the given token and sets the.

[battre, 2016/10/27 11:49:06]

sets the $WORD_MISSING

[Ramin Halavati, 2016/10/27 12:51:20]

Done.

+template <class T>
+void GetLocation(const T* token,
+                 const MatchFinder::MatchResult& result,
+                 Collector::Instance::Location* location) {
+  clang::SourceLocation source_location = token->getLocStart();
+  location->file_name = result.SourceManager->getFilename(source_location);
+  location->line_number =
+      result.SourceManager->getSpellingLineNumber(source_location);
+}
+
+// This class implements the call back functions for AST Matchers. The matchers
+// are defined in RunMatchers function and when a pattern is found there,
+// the run function in this class is called back with information on the match
+// location and description of the match pattern.
+class NetworkAnnotationTagCallback : public MatchFinder::MatchCallback {
+ public:
+  explicit NetworkAnnotationTagCallback(Collector* collector)
+      : collector_(collector) {}
+  ~NetworkAnnotationTagCallback() override = default;
+
+  // Is called on any pattern found by ASTMathers that are defined in RunMathers
+  // function.
+  virtual void run(const MatchFinder::MatchResult& result) override {
+    if (const clang::VarDecl* var_decl =
+            result.Nodes.getNodeAs<clang::VarDecl>("annotation_variable")) {
+      AddVariable(var_decl, result);
+    } else if (const clang::CallExpr* call_expr =
+                   result.Nodes.getNodeAs<clang::CallExpr>("user_function")) {
+      AddFunction(call_expr, result);
+    }
+  }
+
+  // Stores an annotation variable defintion.
+  void AddVariable(const clang::VarDecl* var_decl,
+                   const MatchFinder::MatchResult& result) {
+    Collector::Instance instance;
+
+    GetLocation(var_decl, result, &instance.location);
+    instance.location.object_name = var_decl->getQualifiedNameAsString();
+    instance.variable_reference = clang::dyn_cast<clang::NamedDecl>(var_decl);
+
+    // Mark it as transitive parameter if it doesn't have initialization but
+    // it's a function parameter. Otherwise, extract it's content.
+    if (!var_decl->hasInit() && var_decl->isLocalVarDeclOrParm() &&
+        !var_decl->isLocalVarDecl()) {
+      instance.transitive_parameter = true;
+    } else if (auto* init_expr = var_decl->getInit()) {
+      if (auto* call_expr = clang::dyn_cast<clang::CallExpr>(init_expr))
+        GetAnnotationText(call_expr, *result.SourceManager, &instance);
+    }
+    // If nothing is set, issue an error.
+    if (!instance.transitive_parameter &&
+        instance.annotation.unique_id.empty() && instance.error.empty()) {
+      instance.error = "Could not resolve variable initialization.";
+    }
+
+    collector_->variable_definitions.push_back(instance);
+  }
+
+  // Stores a function call that uses annotation variables.
+  void AddFunction(const clang::CallExpr* call_expr,
+                   const MatchFinder::MatchResult& result) {
+    Collector::Instance instance;
+
+    GetLocation(call_expr, result, &instance.location);
+    instance.location.function_name =
+        GetCoveringFunction(clang::dyn_cast<clang::Stmt>(call_expr), result);
+    instance.location.object_name =
+        call_expr->getDirectCallee()->getQualifiedNameAsString();
+
+    // Get annotation text.
+    const clang::FunctionDecl* function_decl = call_expr->getDirectCallee();
+    unsigned params_count = function_decl->getNumParams();
+    unsigned args_count = call_expr->getNumArgs();
+
+    for (unsigned i = 0; i < params_count; i++) {
+      if (net_match(clang::QualType::getAsString(
+                        function_decl->getParamDecl(i)->getType().split()),
+                    "NetworkTrafficAnnotationTag")) {
+        if (i >= args_count) {
+          instance.error = "Function missing annotation argument.";
+        } else {
+          // Get the argument.
+          const clang::Expr* arg = call_expr->getArgs()[i];
+
+          // Is it a call to annotate function?
+          if (auto* inner_call_expr = clang::dyn_cast<clang::CallExpr>(arg)) {
+            instance.is_direct_call = true;
+            GetAnnotationText(inner_call_expr, *result.SourceManager,
+                              &instance);
+            instance.error = "";
+          } else {
+            // Then it's a variable.
+            instance.is_direct_call = false;
+            if (auto* pure_arg =
+                    clang::dyn_cast<clang::DeclRefExpr>(arg->IgnoreCasts())) {
+              instance.variable_reference = pure_arg->getFoundDecl();
+              instance.error = "";
+            } else {
+              instance.error = "Unknwon parameter type.";
+            }
+          }
+        }
+        collector_->calls.push_back(instance);
+      }
+    }
+  }
+
+ private:
+  Collector* collector_;
+};
+
+// Sets up ASTMatchers and runs clang tool to populate collector. Returns the
+// result of running the clang tool.
+int RunMatchers(clang::tooling::ClangTool* clang_tool, Collector* collector) {
+  NetworkAnnotationTagCallback call_back(collector);
+  MatchFinder match_finder;
+
+  // Set up a pattern to find variables defined with type
+  // [net::]NetworkTrafficAnnotationTag.
+  match_finder.addMatcher(
+      varDecl(anyOf(hasType(asString("NetworkTrafficAnnotationTag")),
+                    hasType(asString("net::NetworkTrafficAnnotationTag"))))
+          .bind("annotation_variable"),
+      &call_back);
+
+  // Set up a pattern to find functions that have a parameter of type
+  // [net::]NetworkTrafficAnnotationTag.
+  match_finder.addMatcher(
+      callExpr(hasDeclaration(functionDecl(hasAnyParameter(anyOf(
+                   hasType(asString("NetworkTrafficAnnotationTag")),
+                   hasType(asString("net::NetworkTrafficAnnotationTag")))))))
+          .bind("user_function"),
+      &call_back);
+
+  std::unique_ptr<clang::tooling::FrontendActionFactory> frontend_factory =
+      clang::tooling::newFrontendActionFactory(&match_finder);
+  return clang_tool->run(frontend_factory.get());
+}
+
+}  // namespace
+
+int main(int argc, const char* argv[]) {
+  llvm::cl::OptionCategory category("Network Request Audit Extractor Tool");
+  clang::tooling::CommonOptionsParser options(argc, argv, category);
+  clang::tooling::ClangTool tool(options.getCompilations(),
+                                 options.getSourcePathList());
+  Collector collector;
+
+  // Find output folder.
+  const std::string kOutputSpecifier("output_dir=");
+  std::string output_dir;
+  for (int i = 0; i < argc; i++) {
+    if (!strncmp(argv[i], kOutputSpecifier.c_str(),
+                 kOutputSpecifier.length())) {
+      output_dir = argv[i] + kOutputSpecifier.length();
+      break;
+    }
+  }
+
+  if (output_dir == "") {
+    llvm::errs() << "Temporary files directory is not specified.";
+    return -1;
+  }
+
+  int result = RunMatchers(&tool, &collector);
+
+  if (result != 0)
+    return result;
+
+  llvm::outs() << "==== BEGIN EDITS ====\n";
+  llvm::outs() << "==== END EDITS ====\n";
+
+  // For each call, if the parameter is not generated by a direct call to
+  // "DefineNetworkTrafficAnnotation", find the variable that holds the value.
+  for (auto& c : collector.calls) {
+    if (!c.is_direct_call) {
+      // Find the variable.
+      for (const auto& v : collector.variable_definitions)
+        if (v.variable_reference == c.variable_reference) {
+          c.annotation = v.annotation;
+          c.transitive_parameter = v.transitive_parameter;
+          c.error = c.error + (c.error.length() ? "\n+" : "") + v.error;
+          break;
+        }
+      if (!c.annotation.unique_id.length())
+        c.error = "Variable not found.";
+    }
+
+    // If the function just receives the variable and passes it to another
+    // function, ignore it, otherwise write it to file.
+    if (!c.transitive_parameter) {
+      std::string s = c.location.file_name;
+      std::replace(s.begin(), s.end(), '/', '_');
+      std::replace(s.begin(), s.end(), '.', '_');
+      std::string filename = output_dir + "/" + s + "(" +
+                             std::to_string(c.location.line_number) + ").txt";
+
+      std::ofstream output_file(filename);
+      if (output_file.is_open()) {
+        output_file << c.location.file_name << "\n";
+        output_file << c.location.function_name << "\n";
+        output_file << c.location.line_number << "\n";
+        output_file << c.location.object_name << "\n";
+        output_file << c.error << "\n";
+        output_file << c.annotation.unique_id << "\n";
+        output_file << c.annotation.text << "\n";
+        output_file.close();
+      } else {
+        llvm::errs() << "Could not write to file: " << filename << " because "
+                     << strerror(errno) << "\n";
+        return 1;
+      }
+    }
+  }
+
+  return 0;
+}