OLD | NEW |
(Empty) | |
| 1 // Copyright 2017 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 // See README.md for information and build instructions. |
| 6 |
| 7 #include "base/command_line.h" |
| 8 #include "base/files/file_enumerator.h" |
| 9 #include "base/files/file_path.h" |
| 10 #include "base/files/file_util.h" |
| 11 #include "base/files/scoped_temp_dir.h" |
| 12 #include "base/logging.h" |
| 13 #include "base/process/launch.h" |
| 14 #include "base/strings/string_number_conversions.h" |
| 15 #include "base/strings/string_split.h" |
| 16 #include "base/strings/stringprintf.h" |
| 17 #include "net/traffic_annotation/network_traffic_annotation.h" |
| 18 #include "third_party/protobuf/src/google/protobuf/text_format.h" |
| 19 |
| 20 #include "tools/traffic_annotation/traffic_annotation.pb.h" |
| 21 |
| 22 namespace { |
| 23 |
| 24 // Holds an instance of network traffic annotation. |
| 25 struct AnnotationInstance { |
| 26 AnnotationInstance() : state(STATE_OK) {} |
| 27 |
| 28 // Protobuf of the annotation. |
| 29 traffic_annotation::NetworkTrafficAnnotation proto; |
| 30 // Unique id of the annotation. |
| 31 std::string unique_id; |
| 32 // The function that is called with this annotation. |
| 33 std::string callee; |
| 34 // State of this annotation. |
| 35 enum State { STATE_OK, STATE_DUPLICATE, STATE_UNDEFINED } state; |
| 36 }; |
| 37 |
| 38 // Runs clang tool, returns true if all steps are successful. |
| 39 bool RunClangTool(const base::FilePath& src_dir, |
| 40 const base::FilePath& build_dir, |
| 41 const base::FilePath& output_dir, |
| 42 const base::CommandLine::StringVector& path_filters) { |
| 43 #ifdef _WIN32 |
| 44 // Run generate_win_compdb.py on Windows to repair compdb before running clang |
| 45 // tool. |
| 46 base::CommandLine::StringVector argv; |
| 47 argv.push_back("python"); |
| 48 argv.push_back( |
| 49 src_dir.AppendASCII("tools/clang/scripts/generate_win_compdb.py") |
| 50 .MaybeAsASCII()); |
| 51 argv.push_back(build_dir.MaybeAsASCII()); |
| 52 int exit_code = -1; |
| 53 base::Process process = |
| 54 base::LaunchProcess(base::CommandLine(argv), base::LaunchOptions()); |
| 55 if (!process.IsValid() || !process.WaitForExit(&exit_code) || exit_code) { |
| 56 LOG(INFO) << " Executing generate_win_compdb failed.\n"; |
| 57 return false; |
| 58 } |
| 59 #endif |
| 60 |
| 61 for (auto& path : path_filters) { |
| 62 // Create commandline based on OS. |
| 63 base::CommandLine::StringVector argv; |
| 64 #ifdef _WIN32 |
| 65 argv.push_back("python"); |
| 66 argv.push_back( |
| 67 src_dir.AppendASCII("tools/clang/scripts/run_tool.py").MaybeAsASCII()); |
| 68 #else |
| 69 argv.push_back( |
| 70 src_dir.AppendASCII("tools/clang/scripts/run_tool.py").MaybeAsASCII()); |
| 71 argv.push_back("--generate-compdb"); |
| 72 #endif |
| 73 argv.push_back("traffic_annotation_extractor"); |
| 74 argv.push_back(build_dir.MaybeAsASCII()); |
| 75 argv.push_back(path); // TODO(rhalavati@): CHECK ON WINDOWS FOR UNICODE |
| 76 argv.push_back(base::StringPrintf("--tool-args=%s", |
| 77 output_dir.MaybeAsASCII().c_str())); |
| 78 |
| 79 int exit_code = -1; |
| 80 base::Process process = |
| 81 base::LaunchProcess(base::CommandLine(argv), base::LaunchOptions()); |
| 82 if (!process.IsValid() || !process.WaitForExit(&exit_code) || exit_code) { |
| 83 LOG(ERROR) << "Executing clang tool failed.\n"; |
| 84 return false; |
| 85 } |
| 86 } |
| 87 return true; |
| 88 } |
| 89 |
| 90 // Reads an extracted annotation file and returns it in the output variables. |
| 91 // The file starts with six |header_lines| with the following meaning: |
| 92 // 0- File path. |
| 93 // 1- Name of the function that is called using annotation. |
| 94 // 2- Line number. |
| 95 // 3- Name of the function including this position. |
| 96 // 4- Error text associated with this annotation. |
| 97 // 5- Unique id of annotation. |
| 98 // The rest of the file is the protobuf text (|msg_text|). |
| 99 bool ReadFile(const base::FilePath& file_path, |
| 100 std::vector<std::string>* header_lines, |
| 101 std::string* msg_text) { |
| 102 std::string file_content; |
| 103 if (!base::ReadFileToString(file_path, &file_content)) |
| 104 return false; |
| 105 |
| 106 header_lines->clear(); |
| 107 msg_text->clear(); |
| 108 |
| 109 std::vector<std::string> tokens = base::SplitString( |
| 110 file_content, "\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_ALL); |
| 111 |
| 112 // If enough data is extracted, populate outputs, otherwise leave them blank. |
| 113 if (tokens.size() > 6) { |
| 114 for (int i = 0; i < 6; i++) |
| 115 header_lines->push_back(tokens[i]); |
| 116 for (size_t i = 6; i < tokens.size(); i++) |
| 117 *msg_text += tokens[i] + "\n"; |
| 118 } |
| 119 |
| 120 return true; |
| 121 } |
| 122 |
| 123 // Reads all extracted txt files from given input folder and populates instances |
| 124 // and errors. Errors include not finding the file, incorrect content, or error |
| 125 // passed from clang tool. |
| 126 void ReadExtractedFiles(const base::FilePath& folder_name, |
| 127 std::vector<AnnotationInstance>* instances, |
| 128 std::vector<std::string>* errors) { |
| 129 #ifdef _WIN32 |
| 130 base::FileEnumerator file_iter(folder_name, false, |
| 131 base::FileEnumerator::FILES, L"*.txt"); |
| 132 #else |
| 133 base::FileEnumerator file_iter(folder_name, false, |
| 134 base::FileEnumerator::FILES, "*.txt"); |
| 135 #endif |
| 136 while (!file_iter.Next().empty()) { |
| 137 std::string file_name = file_iter.GetInfo().GetName().AsUTF8Unsafe(); |
| 138 LOG(INFO) << "\tReading " << file_name.c_str() << "...\n"; |
| 139 |
| 140 std::vector<std::string> header_lines; |
| 141 std::string msg_text; |
| 142 if (!ReadFile(folder_name.Append(file_iter.GetInfo().GetName()), |
| 143 &header_lines, &msg_text)) { |
| 144 errors->push_back( |
| 145 base::StringPrintf("Could not open file '%s'\n.", file_name.c_str())); |
| 146 continue; |
| 147 } |
| 148 if (header_lines.size() < 6) { |
| 149 errors->push_back(base::StringPrintf( |
| 150 "Header lines are not complete for file '%s'\n.", file_name.c_str())); |
| 151 continue; |
| 152 } |
| 153 if (!header_lines[4].empty()) { |
| 154 errors->push_back( |
| 155 base::StringPrintf("Error passed from extractor for file '%s': %s\n.", |
| 156 file_name.c_str(), header_lines[4].c_str())); |
| 157 continue; |
| 158 } |
| 159 |
| 160 AnnotationInstance new_instance; |
| 161 if (header_lines[5] == "\"Undefined\"") { |
| 162 new_instance.state = AnnotationInstance::STATE_UNDEFINED; |
| 163 } else if (!google::protobuf::TextFormat::ParseFromString( |
| 164 msg_text, (google::protobuf::Message*)&new_instance)) { |
| 165 errors->push_back(base::StringPrintf( |
| 166 "Could not parse protobuf for file '%s'\n.", file_name.c_str())); |
| 167 continue; |
| 168 } |
| 169 |
| 170 // Add header data to new instance. |
| 171 traffic_annotation::NetworkTrafficAnnotation_TrafficSource* src = |
| 172 new_instance.proto.mutable_source(); |
| 173 src->set_file(header_lines[0]); |
| 174 src->set_function(header_lines[1]); |
| 175 int line; |
| 176 base::StringToInt(header_lines[2], &line); |
| 177 src->set_line(line); |
| 178 new_instance.callee = header_lines[3]; |
| 179 new_instance.unique_id = header_lines[5]; |
| 180 instances->push_back(new_instance); |
| 181 } |
| 182 LOG(INFO) << instances->size() << " annotation instance(s) read.\n"; |
| 183 } |
| 184 |
| 185 // Checks to see if unique ids are really unique and marks the ones which are |
| 186 // not. |
| 187 void MarkRepeatedUniqueIds(std::vector<AnnotationInstance>* instances) { |
| 188 std::map<std::string, AnnotationInstance*> unique_ids; |
| 189 for (auto& instance : *instances) { |
| 190 if (instance.state == AnnotationInstance::STATE_OK) { |
| 191 auto match = unique_ids.find(instance.unique_id); |
| 192 if (match != unique_ids.end()) |
| 193 instance.state = match->second->state = |
| 194 AnnotationInstance::STATE_DUPLICATE; |
| 195 else |
| 196 unique_ids.insert( |
| 197 std::make_pair(std::string(instance.unique_id), &instance)); |
| 198 } |
| 199 } |
| 200 } |
| 201 |
| 202 // Writes summary of annotation instances to a file. Returns true if successful. |
| 203 bool WriteSummaryFile(const std::vector<AnnotationInstance>& instances, |
| 204 const std::vector<std::string>& errors, |
| 205 const base::FilePath& file_path) { |
| 206 std::string report = ""; |
| 207 |
| 208 if (errors.size()) { |
| 209 report = "[Errors]\n"; |
| 210 |
| 211 for (const auto& error : errors) |
| 212 report += error + "\n"; |
| 213 } |
| 214 |
| 215 report += "[Annotations]\n"; |
| 216 |
| 217 for (const auto& instance : instances) { |
| 218 report += |
| 219 "------------------------------------------------------------" |
| 220 "--------------------\n"; |
| 221 report += base::StringPrintf("Unique ID: %s\n", instance.unique_id.c_str()); |
| 222 if (instance.state == AnnotationInstance::STATE_UNDEFINED) |
| 223 report += base::StringPrintf("WARNING: Undefined annotation.\n"); |
| 224 else if (instance.state == AnnotationInstance::STATE_DUPLICATE) |
| 225 report += base::StringPrintf("WARNING: Duplicate unique id.\n"); |
| 226 report += |
| 227 base::StringPrintf("Calling Function: %s\n", instance.callee.c_str()); |
| 228 std::string temp; |
| 229 google::protobuf::TextFormat::PrintToString(instance.proto, &temp); |
| 230 report += base::StringPrintf("Content:\n%s\n", temp.c_str()); |
| 231 } |
| 232 |
| 233 if (base::WriteFile(file_path, report.c_str(), report.length()) == -1) { |
| 234 LOG(INFO) << " Could not create output file: " |
| 235 << file_path.MaybeAsASCII().c_str() << ".\n"; |
| 236 return false; |
| 237 } |
| 238 |
| 239 LOG(INFO) << "Output file " << file_path.MaybeAsASCII().c_str() |
| 240 << " written for " << instances.size() << " instances.\n"; |
| 241 return true; |
| 242 } |
| 243 |
| 244 } // namespace |
| 245 |
| 246 #ifdef _WIN32 |
| 247 int wmain(int argc, wchar_t* argv[]) { |
| 248 #else |
| 249 int main(int argc, char* argv[]) { |
| 250 #endif |
| 251 // Parse switches. |
| 252 base::CommandLine command_line = base::CommandLine(argc, argv); |
| 253 if (command_line.HasSwitch("help") || command_line.HasSwitch("h") || |
| 254 argc == 1) { |
| 255 LOG(INFO) |
| 256 << "Usage: traffic_annotation_auditor [OPTION]... [path_filter]...\n" |
| 257 "Extracts network traffic annotations from source files. If path " |
| 258 "filter(s) are specified, only those directories of the source " |
| 259 "will be analyzed.\n" |
| 260 "Options:\n" |
| 261 " -h, --help Shows help.\n" |
| 262 " --build-dir Path to the build directory from which " |
| 263 "the\n" |
| 264 " annotations will be extracted.\n" |
| 265 " --extractor-output-dir Path to the directory that extracted\n" |
| 266 " partial files will be written to. " |
| 267 "Will\n" |
| 268 " be automatically generated and " |
| 269 "deleted\n" |
| 270 " if not specified.\n" |
| 271 " --extracted-input-dir Path to a directory where extracted\n" |
| 272 " partial annotations are already " |
| 273 "stored.\n" |
| 274 " If specified, build directory will be\n" |
| 275 " ignored.\n" |
| 276 " --summary-file Path to an output file with summary of\n" |
| 277 " extracted annotations.\n" |
| 278 "Example:\n" |
| 279 " traffic_annotation_auditor --build-dir=out/Debug\n" |
| 280 " --summary-file=report.txt\n"; |
| 281 return 1; |
| 282 } |
| 283 base::FilePath extracted_files_dir = |
| 284 command_line.GetSwitchValuePath("extracted-input-dir"); |
| 285 base::FilePath build_dir = command_line.GetSwitchValuePath("build-dir"); |
| 286 base::FilePath extractor_output_dir = |
| 287 command_line.GetSwitchValuePath("extractor-output-dir"); |
| 288 base::FilePath summary_file = command_line.GetSwitchValuePath("summary-file"); |
| 289 base::ScopedTempDir temp_dir; |
| 290 |
| 291 if (summary_file.empty()) |
| 292 LOG(ERROR) << "WARNING: Output file not specified.\n"; |
| 293 |
| 294 // Extract annotations. |
| 295 if (extracted_files_dir.empty()) { |
| 296 // Get build directory, if it is empty issue an error. |
| 297 if (build_dir.empty()) { |
| 298 LOG(ERROR) |
| 299 << "You must either specify build directory to run clang tool and " |
| 300 "extract annotations, or specify extracted files's input " |
| 301 "directory where already extracted files exist.\n"; |
| 302 return 1; |
| 303 } |
| 304 // If output directory is not provided, create a temporary one. |
| 305 if (extractor_output_dir.empty()) { |
| 306 if (!temp_dir.CreateUniqueTempDirUnderPath(build_dir)) { |
| 307 LOG(ERROR) << "Could not create temporary directory in " |
| 308 << build_dir.MaybeAsASCII().c_str() << ".\n"; |
| 309 return 1; |
| 310 } |
| 311 extractor_output_dir = temp_dir.GetPath(); |
| 312 } else { |
| 313 // Ensure given directory is empty. |
| 314 if (!base::FileEnumerator(extractor_output_dir, false, |
| 315 base::FileEnumerator::FILES) |
| 316 .Next() |
| 317 .empty()) { |
| 318 LOG(ERROR) << "Output directory " |
| 319 << extractor_output_dir.MaybeAsASCII().c_str() |
| 320 << "should be empty .\n"; |
| 321 return 1; |
| 322 } |
| 323 } |
| 324 |
| 325 // Get path filters, if none is provided, just add all. |
| 326 base::CommandLine::StringVector path_filters = command_line.GetArgs(); |
| 327 if (!path_filters.size()) { |
| 328 base::FilePath temp; |
| 329 path_filters.push_back(temp.AppendASCII("./").value().c_str()); |
| 330 } |
| 331 |
| 332 // Eexcutable is usually in out/[Build Dir], so the path to source is |
| 333 // extracted by moving two directories up. |
| 334 if (!RunClangTool(command_line.GetProgram().DirName().AppendASCII("../.."), |
| 335 build_dir, extractor_output_dir, path_filters)) { |
| 336 return 1; |
| 337 } |
| 338 |
| 339 extracted_files_dir = extractor_output_dir; |
| 340 } |
| 341 |
| 342 // Read all extracted files. |
| 343 std::vector<AnnotationInstance> instances; |
| 344 std::vector<std::string> errors; |
| 345 ReadExtractedFiles(extracted_files_dir, &instances, &errors); |
| 346 |
| 347 if (instances.empty()) { |
| 348 LOG(ERROR) << "Could not read any file.\n"; |
| 349 return 1; |
| 350 } else { |
| 351 MarkRepeatedUniqueIds(&instances); |
| 352 } |
| 353 |
| 354 // Create Summary file if requested. |
| 355 if (!summary_file.empty() && |
| 356 !WriteSummaryFile(instances, errors, summary_file)) |
| 357 return 1; |
| 358 |
| 359 return 0; |
| 360 } |
OLD | NEW |