Add static domain security state generator tool.

net/tools/domain_security_preload_generator/domain_security_preload_generator.cc

Index: net/tools/domain_security_preload_generator/domain_security_preload_generator.cc
diff --git a/net/tools/domain_security_preload_generator/domain_security_preload_generator.cc b/net/tools/domain_security_preload_generator/domain_security_preload_generator.cc
new file mode 100644
index 0000000000000000000000000000000000000000..f60183dc12171fd229b4840fc854ed05048fc9b4
--- /dev/null
+++ b/net/tools/domain_security_preload_generator/domain_security_preload_generator.cc
@@ -0,0 +1,532 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <iostream>
+
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "base/command_line.h"
+#include "base/files/file_util.h"
+#include "base/json/json_reader.h"
+#include "base/path_service.h"
+#include "base/strings/string_util.h"
+#include "base/strings/utf_string_conversions.h"
+
+#include "base/values.h"
+#include "net/tools/domain_security_preload_generator/cert_util.h"
+#include "net/tools/domain_security_preload_generator/domain_security_entry.h"
+#include "net/tools/domain_security_preload_generator/pinset.h"
+#include "net/tools/domain_security_preload_generator/pinsets.h"
+#include "net/tools/domain_security_preload_generator/preloaded_state_generator.h"
+#include "net/tools/domain_security_preload_generator/spki_hash.h"
+
+using net::DomainSecurityEntry;
+using net::DomainSecurityEntries;
+using net::Pinset;
+using net::Pinsets;
+using net::PreloadedStateGenerator;
+using net::DomainIDList;
+using net::SPKIHash;
+
+namespace {
+
+// Print the command line help.
+void PrintHelp() {
+  std::cout << "domain_security_preload_generator [-v] <json-file> <pins-file>"
+            << "<template-file> <output-file>" << std::endl;
+}
+
+// Parses the |json| string and copies the items under the entries key to
+// |entries| the pinsets under the pinset key to |pinsets| and the domain IDs

[agl, 2016/12/06 18:51:35]

comma after "|entries|" and quotes around the key names.

[martijnc, 2016/12/07 22:37:53]

Done.

+// under the domain_ids key to |domain_ids|.
+//
+// More info about the format can be found in
+// net/http/transport_security_state_static.json
+bool ParseJSON(const std::string& json,
+               DomainSecurityEntries* entries,
+               Pinsets* pinsets,
+               DomainIDList* domain_ids) {
+  std::unique_ptr<base::Value> value = base::JSONReader::Read(json);
+  base::DictionaryValue* dict_value = nullptr;
+  if (!value.get() || !value->GetAsDictionary(&dict_value)) {
+    std::cerr << "Failed parsing JSON" << std::endl;
+    return false;
+  }
+
+  const base::ListValue* preload_entries = nullptr;
+  if (!dict_value->GetList("entries", &preload_entries)) {
+    std::cerr << "Failed parsing JSON (entries)" << std::endl;
+    return false;
+  }
+
+  for (size_t i = 0; i < preload_entries->GetSize(); ++i) {
+    const base::DictionaryValue* parsed = nullptr;
+    if (!preload_entries->GetDictionary(i, &parsed)) {
+      std::cerr << "Could not parse entry " << i << std::endl;
+      return false;
+    }
+
+    std::string hostname = "";

[agl, 2016/12/06 18:51:35]

no need to say = "";

[martijnc, 2016/12/07 22:37:53]

Removed.

+    if (!parsed->GetString("name", &hostname)) {
+      std::cerr << "Could extract name from entry " << i << std::endl;

[agl, 2016/12/06 18:51:35]

s/Could/Couldn't/

[martijnc, 2016/12/07 22:37:53]

Done.

+      return false;
+    }
+
+    bool include_subdomains = false;
+    parsed->GetBoolean("include_subdomains", &include_subdomains);
+
+    std::string mode = "";

[agl, 2016/12/06 18:51:35]

ditto about = "" and elsewhere in this function.

[martijnc, 2016/12/07 22:37:53]

Removed.

+    parsed->GetString("mode", &mode);
+
+    bool hpkp_include_subdomains = false;
+    parsed->GetBoolean("include_subdomains_for_pinning",
+                       &hpkp_include_subdomains);
+
+    std::string pinset = "";
+    parsed->GetString("pins", &pinset);
+
+    bool expect_ct = false;
+    parsed->GetBoolean("expect_ct", &expect_ct);
+
+    std::string expect_ct_report_uri;
+    parsed->GetString("expect_ct_report_uri", &expect_ct_report_uri);
+
+    bool expect_staple = false;
+    parsed->GetBoolean("expect_staple", &expect_staple);
+
+    bool expect_staple_include_subdomains = false;
+    parsed->GetBoolean("include_subdomains_for_expect_staple",
+                       &expect_staple_include_subdomains);
+
+    std::string expect_staple_report_uri = "";
+    parsed->GetString("expect_staple_report_uri", &expect_staple_report_uri);
+
+    std::unique_ptr<DomainSecurityEntry> entry(new DomainSecurityEntry(

[agl, 2016/12/06 18:51:35]

It seems that DomainSecurityEntry might be better as a struct. The constructor
really has too many argument and, if it were a struct, the above methods on
|parsed| could write into it directly.

[martijnc, 2016/12/07 22:37:53]

Done.

+        hostname, include_subdomains, mode == "force-https",
+        hpkp_include_subdomains, pinset, expect_ct, expect_ct_report_uri,
+        expect_staple, expect_staple_include_subdomains,
+        expect_staple_report_uri));
+
+    entries->push_back(std::move(entry));
+  }
+
+  const base::ListValue* pinsets_list = nullptr;
+  if (!dict_value->GetList("pinsets", &pinsets_list)) {
+    std::cerr << "Failed parsing JSON (pinsets)" << std::endl;
+    return false;
+  }
+
+  for (size_t i = 0; i < pinsets_list->GetSize(); ++i) {
+    const base::DictionaryValue* parsed = nullptr;
+    if (!pinsets_list->GetDictionary(i, &parsed)) {
+      std::cerr << "Could not parse pinset " << i << "; skipping entry"
+                << std::endl;
+      continue;
+    }
+
+    std::string name;
+    if (!parsed->GetString("name", &name)) {
+      continue;
+    }
+
+    std::string report_uri;
+    parsed->GetString("report_uri", &report_uri);
+
+    std::unique_ptr<Pinset> pinset(new Pinset(name, report_uri));
+
+    const base::ListValue* pinset_static_hashes_list = nullptr;
+    if (parsed->GetList("static_spki_hashes", &pinset_static_hashes_list)) {
+      for (size_t i = 0; i < pinset_static_hashes_list->GetSize(); ++i) {
+        std::string hash;
+        pinset_static_hashes_list->GetString(i, &hash);
+        pinset->AddStaticSPKIHash(hash);
+      }
+    }
+
+    const base::ListValue* pinset_bad_static_hashes_list = nullptr;
+    if (parsed->GetList("bad_static_spki_hashes",
+                        &pinset_bad_static_hashes_list)) {
+      for (size_t i = 0; i < pinset_bad_static_hashes_list->GetSize(); ++i) {
+        std::string hash;
+        pinset_bad_static_hashes_list->GetString(i, &hash);
+        pinset->AddBadStaticSPKIHash(hash);
+      }
+    }
+
+    pinsets->RegisterPinset(std::move(pinset));
+  }
+
+  const base::ListValue* domain_ids_list = nullptr;
+  if (!dict_value->GetList("domain_ids", &domain_ids_list)) {
+    std::cerr << "Failed parsing JSON (domain_ids)" << std::endl;
+    return false;
+  }
+
+  for (size_t i = 0; i < domain_ids_list->GetSize(); ++i) {
+    std::string domain;
+    domain_ids_list->GetString(i, &domain);
+    domain_ids->push_back(domain);
+  }
+
+  return true;
+}
+
+// Returns true if |candidate| only contains characters in the range a-Z and 0-9
+// and false otherwise.
+bool IsValidName(const std::string& candidate) {
+  bool isValid = true;

[agl, 2016/12/06 18:51:35]

If this is matching
https://github.com/chromium/hstspreload/blob/master/cmd/transport_security_st...,
then it needs to ensure !candidate.empty() and that candidate[0] in 'A'..'Z'.

[martijnc, 2016/12/07 22:37:53]

Done.

+  for (const char& character : candidate) {
+    isValid = (character >= '0' && character <= '9') ||
+              (character >= 'a' && character <= 'z') ||
+              (character >= 'A' && character <= 'Z') || character == '_';
+    if (!isValid) {
+      return false;
+    }
+  }
+  return true;
+}
+
+static const char* kStartOfCert = "-----BEGIN CERTIFICATE";

[agl, 2016/12/06 18:51:35]

ditto about [] vs *

[martijnc, 2016/12/07 22:37:53]

Done.

+static const char* kStartOfPublicKey = "-----BEGIN PUBLIC KEY";
+static const char* kEndOfCert = "-----END CERTIFICATE";
+static const char* kEndOfPublicKey = "-----END PUBLIC KEY";
+static const char* kStartOfSHA256 = "sha256/";
+
+enum class CertificateParserState {
+  PRE_NAME,
+  POST_NAME,
+  IN_CERTIFICATE,
+  IN_PUBLIC_KEY,
+  IN_HASH
+};
+
+// Extracts SPKI information from the preloaded pins file. The SPKI's can be
+// in the form of a PEM certificate, a PEM public key, or a BASE64 string.
+//
+// More info about the format can be found in
+// net/http/transport_security_state_static.pins
+bool ParseCertificatesFile(const std::string& certs_input, Pinsets* pinsets) {
+  std::istringstream input_stream(certs_input);
+  std::string line;
+  CertificateParserState current_state = CertificateParserState::PRE_NAME;
+
+  const base::CompareCase& compare_mode = base::CompareCase::INSENSITIVE_ASCII;
+  std::string name;
+  std::string buffer;
+  SPKIHash hash;
+
+  for (std::string line; std::getline(input_stream, line);) {
+    if (line[0] == '#') {
+      continue;
+    }
+
+    if (!line.size() && current_state == CertificateParserState::PRE_NAME) {

[agl, 2016/12/06 18:51:35]

write !line.size() as line.empty()

[martijnc, 2016/12/07 22:37:53]

Done.

+      continue;
+    }
+
+    switch (current_state) {
+      case CertificateParserState::PRE_NAME:
+        if (!IsValidName(line)) {
+          std::cerr << "Invalid name in certificates file: " << line;
+          return false;
+        }
+        name = line;
+        current_state = CertificateParserState::POST_NAME;
+        break;
+      case CertificateParserState::POST_NAME:
+        if (base::StartsWith(line, kStartOfSHA256, compare_mode)) {
+          buffer = line;
+          current_state = CertificateParserState::IN_HASH;

[agl, 2016/12/06 18:51:35]

The IN_HASH state is a little odd because it means that the following line is
always ignored, no? The Go code processes the hash immediately and that might be
best here.

[martijnc, 2016/12/07 22:37:53]

Done.

+        } else if (base::StartsWith(line, kStartOfCert, compare_mode)) {
+          buffer = line + '\n';
+          current_state = CertificateParserState::IN_CERTIFICATE;
+        } else if (base::StartsWith(line, kStartOfPublicKey, compare_mode)) {
+          buffer = line + '\n';
+          current_state = CertificateParserState::IN_PUBLIC_KEY;
+        } else {
+          std::cerr << "Invalid value in certificates file for " << name
+                    << std::endl;
+          return false;
+        }
+        break;
+      case CertificateParserState::IN_HASH:
+        if (!hash.FromString(buffer)) {
+          std::cerr << "Invalid hash value in certificate file for " << name
+                    << std::endl;
+          return false;
+        }
+
+        pinsets->RegisterSPKIHash(name, hash);
+        current_state = CertificateParserState::PRE_NAME;
+        break;
+      case CertificateParserState::IN_CERTIFICATE:
+        if (line.size()) {

[agl, 2016/12/06 18:51:35]

I don't think you need to ignore blank lines.

[martijnc, 2016/12/07 22:37:53]

Removed.

+          buffer += line + '\n';
+          if (!base::StartsWith(line, kEndOfCert, compare_mode)) {
+            continue;
+          }
+        }
+
+        if (!CalculateSPKIHashFromCertificate(buffer, &hash)) {
+          std::cerr << "Parsing of the certificate \"" << name << "\" failed"
+                    << std::endl;
+          return false;
+        }
+
+        pinsets->RegisterSPKIHash(name, hash);
+        current_state = CertificateParserState::PRE_NAME;
+        break;
+      case CertificateParserState::IN_PUBLIC_KEY:
+        if (line.size()) {

[agl, 2016/12/06 18:51:35]

I don't think you need to ignore blank lines.

[martijnc, 2016/12/07 22:37:53]

Removed.

+          buffer += line + '\n';
+          if (!base::StartsWith(line, kEndOfPublicKey, compare_mode)) {
+            continue;
+          }
+        }
+
+        if (!CalculateSPKIHashFromKey(buffer, &hash)) {
+          std::cerr << "Parsing of the public key \"" << name << "\" failed"
+                    << std::endl;
+          return false;
+        }
+
+        pinsets->RegisterSPKIHash(name, hash);
+        current_state = CertificateParserState::PRE_NAME;
+        break;
+    }

[agl, 2016/12/06 18:51:35]

default:
  CHECK(false) << "Unknown state " << current_state;

(Or something similar.)

[martijnc, 2016/12/07 22:37:53]

Done.

+  }
+
+  return true;
+}
+
+// Checks if there are pins with the same name or the same hash.
+bool CheckForDuplicatePins(const Pinsets& pinsets) {
+  std::set<std::string> seen_names;
+  std::map<std::string, std::string> seen_hashes;
+
+  for (const auto& pin : pinsets.spki_hashes()) {
+    if (seen_names.find(pin.first) != seen_names.cend()) {
+      std::cerr << "Duplicate pin name: " << pin.first << std::endl;
+      return false;
+    }
+    seen_names.insert(pin.first);
+
+    std::string hash =
+        std::string(pin.second.data(), pin.second.data() + (pin.second.size()));

[agl, 2016/12/06 18:51:35]

drop extra parens around |pin.second.size()|.

[martijnc, 2016/12/07 22:37:53]

Done.

+    std::map<std::string, std::string>::iterator it = seen_hashes.find(hash);
+    if (it != seen_hashes.cend()) {
+      std::cerr << "Duplicate pin hash for " << pin.first << " already seen as "
+                << it->second << std::endl;
+      return false;
+    }
+    seen_hashes.insert(std::pair<std::string, std::string>(hash, pin.first));
+  }
+
+  return true;
+}
+
+// Checks whether there are:
+//   - pinsets that reference non-existing pins;
+//   - two pinsets that share the same name;
+//   - there are unused pins.
+bool CheckCertificatesInPinsets(const Pinsets& pinsets) {
+  std::set<std::string> pin_names;
+  for (const auto& pin : pinsets.spki_hashes()) {
+    pin_names.insert(pin.first);
+  }
+
+  std::set<std::string> used_pin_names;
+  std::set<std::string> pinset_names;
+  for (const auto& pinset : pinsets.pinsets()) {
+    if (pinset_names.find(pinset.second->name()) != pinset_names.cend()) {
+      std::cerr << "Duplicate pinset name: " << pinset.second->name()
+                << std::endl;
+      return false;
+    }
+    pinset_names.insert(pinset.second->name());
+
+    const std::vector<std::string>& good_hashes =
+        pinset.second->static_spki_hashes();
+    const std::vector<std::string>& bad_hashes =
+        pinset.second->bad_static_spki_hashes();
+
+    std::vector<std::string> all_pin_names;
+    all_pin_names.reserve(good_hashes.size() + bad_hashes.size());
+    all_pin_names.insert(all_pin_names.end(), good_hashes.begin(),
+                         good_hashes.end());
+    all_pin_names.insert(all_pin_names.end(), bad_hashes.begin(),
+                         bad_hashes.end());
+
+    for (const auto& pin_name : all_pin_names) {
+      if (pin_names.find(pin_name) == pin_names.cend()) {
+        std::cerr << "Unknown pin: " << pin_name << std::endl;
+        return false;
+      }
+      used_pin_names.insert(pin_name);
+    }
+  }
+
+  for (const auto& pin_name : pin_names) {
+    if (used_pin_names.find(pin_name) == used_pin_names.cend()) {
+      std::cerr << "Unused pin: " << pin_name << std::endl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+// Checks if there are two or more entries for the same hostname.
+bool CheckDuplicateEntries(const DomainSecurityEntries& entries) {
+  std::set<std::string> seen_entries;
+  for (const auto& entry : entries) {
+    if (seen_entries.find(entry->hostname()) != seen_entries.cend()) {
+      std::cerr << "Duplicate entry for " << entry->hostname() << std::endl;
+      return false;
+    }
+    seen_entries.insert(entry->hostname());
+  }
+  return true;
+}
+
+// Checks for entries which have no effect.
+bool CheckNoopEntries(const DomainSecurityEntries& entries) {
+  for (const auto& entry : entries) {
+    if (!entry->force_https() && !entry->pinset().size() &&
+        !entry->expect_ct() && !entry->expect_staple()) {
+      if (entry->hostname() == "learn.doubleclick.net") {
+        // This entry is deliberately used as an exclusion.
+        continue;
+      }
+
+      std::cerr
+          << "Entry for " + entry->hostname() +
+                 " has no mode, no pins and is not expect-CT or expect-staple."
+          << std::endl;
+      return false;
+    }
+  }
+  return true;
+}
+
+// Checks all entries for incorrect usage of the includeSubdomains flags.
+bool CheckSubdomainsFlags(const DomainSecurityEntries& entries) {
+  for (const auto& entry : entries) {
+    if (entry->include_subdomains() && entry->hpkp_include_subdomains()) {
+      std::cerr << "Entry for " << entry->hostname()
+                << " sets include_subdomains_for_pinning but also sets "
+                   "include_subdomains, which implies it."
+                << std::endl;
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace
+
+int main(int argc, char* argv[]) {
+  InitializeCryptoLibrary();
+
+  base::CommandLine::Init(argc, argv);
+  const base::CommandLine& command_line =
+      *base::CommandLine::ForCurrentProcess();
+
+#if defined(OS_WIN)
+  std::vector<std::string> args;
+  base::CommandLine::StringVector wide_args = command_line.GetArgs();
+  for (const auto& arg : wide_args) {
+    args.push_back(base::WideToUTF8(arg));
+  }
+#else
+  base::CommandLine::StringVector args = command_line.GetArgs();
+#endif
+  if (args.size() < 4U) {
+    PrintHelp();
+    return 1;
+  }
+
+  bool verbose = command_line.HasSwitch("v");
+
+  base::FilePath json_filepath = base::FilePath::FromUTF8Unsafe(argv[1]);
+  if (!base::PathExists(json_filepath)) {
+    std::cerr << "Input JSON file doesn't exist." << std::endl;
+    return 1;
+  }
+  json_filepath = base::MakeAbsoluteFilePath(json_filepath);
+
+  std::string json_input;
+  if (!base::ReadFileToString(json_filepath, &json_input)) {
+    std::cerr << "Could not read input JSON file." << std::endl;
+    return 1;
+  }
+
+  base::FilePath pins_filepath = base::FilePath::FromUTF8Unsafe(argv[2]);
+  if (!base::PathExists(pins_filepath)) {
+    std::cerr << "Input pins file doesn't exist." << std::endl;
+    return 1;
+  }
+  pins_filepath = base::MakeAbsoluteFilePath(pins_filepath);
+
+  std::string certs_input;
+  if (!base::ReadFileToString(pins_filepath, &certs_input)) {
+    std::cerr << "Could not read input pins file." << std::endl;
+    return 1;
+  }
+
+  DomainSecurityEntries entries;
+  Pinsets pinsets;
+  DomainIDList domain_ids;
+
+  if (!ParseCertificatesFile(certs_input, &pinsets)) {
+    std::cerr << "Error while parsing the pins file." << std::endl;
+    return 1;
+  }
+  if (!ParseJSON(json_input, &entries, &pinsets, &domain_ids)) {
+    std::cerr << "Error while parsing the JSON file." << std::endl;
+    return 1;
+  }
+
+  bool checks_result = CheckDuplicateEntries(entries);
+  checks_result = checks_result && CheckNoopEntries(entries);

[agl, 2016/12/06 18:51:35]

if (!CheckDuplicateEntries(entries) ||
    !CheckNoopEntries(entries) ||
    …) {
  …
  return 1;
}

[martijnc, 2016/12/07 22:37:53]

Done. (`git cl format` formats this differently though)

+  checks_result = checks_result && CheckSubdomainsFlags(entries);
+  checks_result = checks_result && CheckForDuplicatePins(pinsets);
+  checks_result = checks_result && CheckCertificatesInPinsets(pinsets);
+
+  if (!checks_result) {
+    std::cerr << "Checks failed. Aborting." << std::endl;
+    return 1;
+  }
+
+  base::FilePath template_path = base::FilePath::FromUTF8Unsafe(argv[3]);
+  if (!base::PathExists(template_path)) {
+    std::cerr << "Template file doesn't exist." << std::endl;
+    return 1;
+  }
+  template_path = base::MakeAbsoluteFilePath(template_path);
+
+  std::string preload_template;
+  if (!base::ReadFileToString(template_path, &preload_template)) {
+    std::cerr << "Could not read template file." << std::endl;
+    return 1;
+  }
+
+  std::string result;
+  PreloadedStateGenerator generator;
+  result = generator.Generate(preload_template, entries, domain_ids, pinsets,
+                              verbose);
+
+  base::FilePath output_path;
+  output_path = base::FilePath::FromUTF8Unsafe(argv[4]);
+
+  if (base::WriteFile(output_path, result.c_str(), result.size()) <= 0) {
+    std::cerr << "Failed to write output." << std::endl;
+    return 1;
+  }
+
+  return 0;
+}