Check for system include paths by looking at clang's HeaderSearchOptions,

tools/clang/translation_unit/TranslationUnitGenerator.cpp

 // Copyright (c) 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // This implements a Clang tool to generate compilation information that is
 // sufficient to recompile the code with clang. For each compilation unit, all
 // source files which are necessary for compiling it are determined. For each
 // compilation unit, a file is created containing a list of all file paths of
 // included files.
 
 #include <assert.h>
 #include <unistd.h>
 #include <fstream>
 #include <iostream>
 #include <memory>
 #include <set>
 #include <stack>
 #include <string>
 #include <vector>
 
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/Refactoring.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/CommandLine.h"
 
+using clang::HeaderSearchOptions;
 using clang::tooling::CommonOptionsParser;
 using std::set;
 using std::stack;
 using std::string;
 using std::vector;
 
 namespace {
 // Set of preprocessor callbacks used to record files included.
 class IncludeFinderPPCallbacks : public clang::PPCallbacks {
  public:
   IncludeFinderPPCallbacks(clang::SourceManager* source_manager,
                            string* main_source_file,
-                           set<string>* source_file_paths)
-      : source_manager_(source_manager),
-        main_source_file_(main_source_file),
-        source_file_paths_(source_file_paths) {}
+                           set<string>* source_file_paths,
+                           const HeaderSearchOptions* header_search_options);
   void FileChanged(clang::SourceLocation /*loc*/,
                    clang::PPCallbacks::FileChangeReason reason,
                    clang::SrcMgr::CharacteristicKind /*file_type*/,
                    clang::FileID /*prev_fid*/) override;
   void AddFile(const string& path);
   void InclusionDirective(clang::SourceLocation hash_loc,
                           const clang::Token& include_tok,
                           llvm::StringRef file_name,
                           bool is_angled,
                           clang::CharSourceRange range,
                           const clang::FileEntry* file,
                           llvm::StringRef search_path,
                           llvm::StringRef relative_path,
                           const clang::Module* imported) override;
   void EndOfMainFile() override;
 
  private:
+  string DoubleSlashSystemHeaders(const string& search_path,
+                                  const string& relative_path) const;
+
   clang::SourceManager* const source_manager_;
   string* const main_source_file_;
   set<string>* const source_file_paths_;
+  set<string> system_header_prefixes_;
   // The path of the file that was last referenced by an inclusion directive,
   // normalized for includes that are relative to a different source file.
   string last_inclusion_directive_;
   // The stack of currently parsed files. top() gives the current file.
   stack<string> current_files_;
 };
 
+IncludeFinderPPCallbacks::IncludeFinderPPCallbacks(
+    clang::SourceManager* source_manager,
+    string* main_source_file,
+    set<string>* source_file_paths,
+    const HeaderSearchOptions* header_search_options)
+      : source_manager_(source_manager),
+        main_source_file_(main_source_file),
+        source_file_paths_(source_file_paths) {
+  for (const auto& prefix : header_search_options->SystemHeaderPrefixes) {

[Adrian Kuegel, 2016/02/01 09:46:25]

In the documentation this is saying "User-specified system header prefixes".
Does that mean it only includes the user specified system header prefixes (i.e.
all those -isystem flags)? Or are there any defaults? I think in the compile
commands for Chrome we don't explicitly add -isystem flags for standard system
headers.

[Adrian Kuegel, 2016/02/01 09:54:10]

I just asked a team mate, and he told me that it includes also the standard
system headers. So this part is probably fine.

[dsansome, 2016/02/02 06:48:58]

It seems to be empty for me anyway.  All the system headers are in the other
list in the System and ExternCSystem groups.  *shrug*

[Adrian Kuegel, 2016/02/02 08:37:19]

Maybe I misunderstood him, or he was wrong. Anyway, good to know that the
standard header prefixes are there.

+    system_header_prefixes_.insert(prefix.Prefix);
+  }
+
+  for (const auto& entry : header_search_options->UserEntries) {

[Adrian Kuegel, 2016/02/01 09:46:25]

I am not so sure about if this works correctly. If I understand this code, this
deals with libraries that are specified with the -I option. In case of Chrome
that means headers within the Chrome source, and they are specified with
relative path in the -I option. For cross references to work correctly we need
to have an exact match between the path Grok knows, and the path Codesearch
knows. So right now for all headers inside Chrome we find them by relative path,
and that is working fine.

[dsansome, 2016/02/02 06:48:58]

This list has all the paths in, but ones from -I have Group
clang::frontend::Angled which I don't insert into system_header_prefixes_ - see
the comments for IncludeDirGroup in
llvm/tools/clang/include/clang/Lex/HeaderSearchOptions.h.

I ran this and printf'd all the entries.  SystemHeaderPrefixes contained
nothing, UserEntries contained the following:

clang::frontend::Angled:
../..
gen
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/gconf/2
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/dbus-1.0
../../build/linux/debian_wheezy_amd64-sysroot/usr/lib/x86_64-linux-gnu/dbus-1.0/include
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/glib-2.0
../../build/linux/debian_wheezy_amd64-sysroot/usr/lib/x86_64-linux-gnu/glib-2.0/include
../../third_party/protobuf/src
gen/protoc_out
../../third_party/protobuf/src
../../third_party/protobuf
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/glib-2.0
../../build/linux/debian_wheezy_amd64-sysroot/usr/lib/x86_64-linux-gnu/glib-2.0/include
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/nss
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/nspr
../../third_party/boringssl/src/include
../../sdch/open-vcdiff/src
../../third_party/zlib
../../build/linux/debian_wheezy_amd64-sysroot/usr/include/glib-2.0
../../build/linux/debian_wheezy_amd64-sysroot/usr/lib/x86_64-linux-gnu/glib-2.0/include
../../third_party/icu/source/common
../../third_party/icu/source/i18n

clang::frontend::System:
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/x86_64-linux-gnu
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/usr/lib/gcc/x86_64-linux-gnu/4.6/../../../../include/c++/4.6/backward
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/usr/local/include
[snip]/src/third_party/llvm-build/Release+Asserts/bin/../lib/clang/3.9.0/include

clang::frontend::ExternCSystem:
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/usr/include/x86_64-linux-gnu
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/include
[snip]/src/build/linux/debian_wheezy_amd64-sysroot/usr/include

I've added a comment here to explain what's going on.

[Adrian Kuegel, 2016/02/02 08:37:19]

Interesting; that seems to indicate that Chromium builds using a special sysroot
in src/build. I didn't know that, either this has changed recently, or I never
noticed that the absolute paths were pointing to within the Chrome source tree.
Anyway, looks like everything is fine.

+    switch (entry.Group) {
+      case clang::frontend::System:
+      case clang::frontend::ExternCSystem:
+      case clang::frontend::CSystem:
+      case clang::frontend::CXXSystem:
+      case clang::frontend::ObjCSystem:
+      case clang::frontend::ObjCXXSystem:
+      case clang::frontend::After:
+        system_header_prefixes_.insert(entry.Path);
+        break;
+      default:
+        break;
+    }
+  }
+}
+
 void IncludeFinderPPCallbacks::FileChanged(
     clang::SourceLocation /*loc*/,
     clang::PPCallbacks::FileChangeReason reason,
     clang::SrcMgr::CharacteristicKind /*file_type*/,
     clang::FileID /*prev_fid*/) {
   if (reason == clang::PPCallbacks::EnterFile) {
     if (!last_inclusion_directive_.empty()) {
       current_files_.push(last_inclusion_directive_);
     } else {
       current_files_.push(
@@ skipping 41 matching lines @@
     string parent =
         llvm::sys::path::parent_path(current_files_.top().c_str()).str();
 
     // If the file is a top level file ("file.cc"), we normalize to a path
     // relative to "./".
     if (parent.empty() || parent == "/")
       parent = ".";
 
     // Otherwise we take the literal path as we stored it for the current
     // file, and append the relative path.
-    last_inclusion_directive_ = parent + "/" + relative_path.str();
+    last_inclusion_directive_ =
+        DoubleSlashSystemHeaders(parent, relative_path.str());
   } else if (!search_path.empty()) {
-    // We want to be able to extract the search path relative to which the
-    // include statement is defined. Therefore if search_path is an absolute
-    // path (indicating it is most likely a system header) we use "//" as a
-    // separator between the search path and the relative path.
-    last_inclusion_directive_ = search_path.str() +
-        (llvm::sys::path::is_absolute(search_path) ? "//" : "/") +
-        relative_path.str();
+    last_inclusion_directive_ =
+        DoubleSlashSystemHeaders(search_path.str(), relative_path.str());
   } else {
     last_inclusion_directive_ = file_name.str();
   }
   AddFile(last_inclusion_directive_);
 }
 
+string IncludeFinderPPCallbacks::DoubleSlashSystemHeaders(
+    const string& search_path,
+    const string& relative_path) const {
+  // We want to be able to extract the search path relative to which the
+  // include statement is defined. Therefore if search_path is a system header
+  // we use "//" as a separator between the search path and the relative path.
+  const bool is_system_header =
+      system_header_prefixes_.find(search_path) !=
+      system_header_prefixes_.end();
+
+  return search_path + (is_system_header ? "//" : "/") + relative_path;
+}
+
 void IncludeFinderPPCallbacks::EndOfMainFile() {
   const clang::FileEntry* main_file =
       source_manager_->getFileEntryForID(source_manager_->getMainFileID());
   assert(*main_source_file_ == main_file->getName());
   AddFile(main_file->getName());
 }
 
 class CompilationIndexerAction : public clang::PreprocessorFrontendAction {
  public:
   CompilationIndexerAction() {}
@@ skipping 20 matching lines @@
   main_source_file_ = inputs[0].getFile();
 
   Preprocess();
 }
 
 void CompilationIndexerAction::Preprocess() {
   clang::Preprocessor& preprocessor = getCompilerInstance().getPreprocessor();
   preprocessor.addPPCallbacks(llvm::make_unique<IncludeFinderPPCallbacks>(
       &getCompilerInstance().getSourceManager(),
       &main_source_file_,
-      &source_file_paths_));
+      &source_file_paths_,
+      &getCompilerInstance().getHeaderSearchOpts()));
   preprocessor.getDiagnostics().setIgnoreAllWarnings(true);
   preprocessor.SetSuppressIncludeNotFoundError(true);
   preprocessor.EnterMainSourceFile();
   clang::Token token;
   do {
     preprocessor.Lex(token);
   } while (token.isNot(clang::tok::eof));
 }
 
 void CompilationIndexerAction::EndSourceFileAction() {
   std::ofstream out(main_source_file_ + ".filepaths");
-  for (string path : source_file_paths_) {
+  for (const string& path : source_file_paths_) {
     out << path << std::endl;
   }
 }
 }  // namespace
 
 static llvm::cl::extrahelp common_help(CommonOptionsParser::HelpMessage);
 
 int main(int argc, const char* argv[]) {
   llvm::cl::OptionCategory category("TranslationUnitGenerator Tool");
   CommonOptionsParser options(argc, argv, category);
   std::unique_ptr<clang::tooling::FrontendActionFactory> frontend_factory =
       clang::tooling::newFrontendActionFactory<CompilationIndexerAction>();
   clang::tooling::ClangTool tool(options.getCompilations(),
                                  options.getSourcePathList());
   // This clang tool does not actually produce edits, but run_tool.py expects
   // this. So we just print an empty edit block.
   llvm::outs() << "==== BEGIN EDITS ====\n";
   llvm::outs() << "==== END EDITS ====\n";
   return tool.run(frontend_factory.get());
 }