Serialize and deserialize dynamic Expect-CT state

net/http/transport_security_persister.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/http/transport_security_persister.h"
 
 #include <memory>
 #include <utility>
 
 #include "base/base64.h"
 #include "base/bind.h"
+#include "base/feature_list.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/json/json_reader.h"
 #include "base/json/json_writer.h"
 #include "base/location.h"
 #include "base/sequenced_task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/values.h"
 #include "crypto/sha2.h"
@@ skipping 52 matching lines @@
 const char kDynamicSPKIHashesExpiry[] = "dynamic_spki_hashes_expiry";
 const char kDynamicSPKIHashes[] = "dynamic_spki_hashes";
 const char kForceHTTPS[] = "force-https";
 const char kStrict[] = "strict";
 const char kDefault[] = "default";
 const char kPinningOnly[] = "pinning-only";
 const char kCreated[] = "created";
 const char kStsObserved[] = "sts_observed";
 const char kPkpObserved[] = "pkp_observed";
 const char kReportUri[] = "report-uri";
+const char kExpectCTExpiry[] = "expect_ct_expiry";
+const char kExpectCTObserved[] = "expect_ct_observed";
+const char kExpectCTEnforce[] = "expect_ct_enforce";
+const char kExpectCTReportUri[] = "expect_ct_report_uri";
 
 std::string LoadState(const base::FilePath& path) {
   std::string result;
   if (!base::ReadFileToString(path, &result)) {
     return "";
   }
   return result;
 }
 
-}  // namespace
-
-TransportSecurityPersister::TransportSecurityPersister(
-    TransportSecurityState* state,
-    const base::FilePath& profile_path,
-    const scoped_refptr<base::SequencedTaskRunner>& background_runner,
-    bool readonly)
-    : transport_security_state_(state),
-      writer_(profile_path.AppendASCII("TransportSecurity"), background_runner),
-      foreground_runner_(base::ThreadTaskRunnerHandle::Get()),
-      background_runner_(background_runner),
-      readonly_(readonly),
-      weak_ptr_factory_(this) {
-  transport_security_state_->SetDelegate(this);
-
-  base::PostTaskAndReplyWithResult(
-      background_runner_.get(), FROM_HERE,
-      base::Bind(&LoadState, writer_.path()),
-      base::Bind(&TransportSecurityPersister::CompleteLoad,
-                 weak_ptr_factory_.GetWeakPtr()));
+bool IsDynamicExpectCTEnabled() {
+  return base::FeatureList::IsEnabled(
+      TransportSecurityState::kDynamicExpectCTFeature);
 }
 
-TransportSecurityPersister::~TransportSecurityPersister() {
-  DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
+// Populates |host| with default values for the STS and PKP states.
+// These default values represent "null" states and are only useful to keep
+// the entries in the resulting JSON consistent. The deserializer will ignore
+// "null" states.
+// TODO(davidben): This can be removed when the STS and PKP states are stored

[mattm, 2017/04/15 04:45:01]

I don't know the history here, but is there anything that can be done here
instead of just continuing the current course?

Is the end goal to have them in different files or just a nicer organization
within a single file? If it's the first, then we could just put expect-CT data
in a new file right from the start. If it's the latter then it may be a bit more
complex.. maybe store all the expect-ct keys in a sub-dictionary under each
hostname instead?

+// independently on disk. https://crbug.com/470295
+void PopulateEntryWithDefaults(base::DictionaryValue* host) {
+  host->Clear();
 
-  if (writer_.HasPendingWrite())
-    writer_.DoScheduledWrite();
+  // STS default values.
+  host->SetBoolean(kStsIncludeSubdomains, false);
+  host->SetDouble(kStsObserved, 0.0);
+  host->SetDouble(kExpiry, 0.0);
+  host->SetString(kMode, kDefault);
 
-  transport_security_state_->SetDelegate(NULL);
+  // PKP default values.
+  host->SetBoolean(kPkpIncludeSubdomains, false);
+  host->SetDouble(kPkpObserved, 0.0);
+  host->SetDouble(kDynamicSPKIHashesExpiry, 0.0);
+
+  // Expect-CT default values.

[mattm, 2017/04/15 04:45:01]

Should kExpectCTReportUri be in here?

[estark, 2017/04/15 20:18:03]

Removed this section per your comment below.

+  if (IsDynamicExpectCTEnabled()) {
+    host->SetBoolean(kExpectCTObserved, 0.0);
+    host->SetBoolean(kExpectCTExpiry, 0.0);
+    host->SetBoolean(kExpectCTEnforce, false);
+  }
 }
 
-void TransportSecurityPersister::StateIsDirty(
-    TransportSecurityState* state) {
-  DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
-  DCHECK_EQ(transport_security_state_, state);
-
-  if (!readonly_)
-    writer_.ScheduleWrite(this);
-}
-
-bool TransportSecurityPersister::SerializeData(std::string* output) {
-  DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
-
-  base::DictionaryValue toplevel;
-  base::Time now = base::Time::Now();
-
-  // TODO(davidben): Fix the serialization format by splitting the on-disk
-  // representation of the STS and PKP states. https://crbug.com/470295.
-  TransportSecurityState::STSStateIterator sts_iterator(
-      *transport_security_state_);
+// Serializes STS data from |state| into |toplevel|. Any existing state in
+// |toplevel| for each item is overwritten.
+void SerializeSTSData(TransportSecurityState* state,
+                      base::DictionaryValue* toplevel) {
+  TransportSecurityState::STSStateIterator sts_iterator(*state);
   for (; sts_iterator.HasNext(); sts_iterator.Advance()) {
     const std::string& hostname = sts_iterator.hostname();
     const TransportSecurityState::STSState& sts_state =
         sts_iterator.domain_state();
 
     const std::string key = HashedDomainToExternalString(hostname);
     std::unique_ptr<base::DictionaryValue> serialized(
         new base::DictionaryValue);
     PopulateEntryWithDefaults(serialized.get());
 
     serialized->SetBoolean(kStsIncludeSubdomains, sts_state.include_subdomains);
     serialized->SetDouble(kStsObserved, sts_state.last_observed.ToDoubleT());
     serialized->SetDouble(kExpiry, sts_state.expiry.ToDoubleT());
 
     switch (sts_state.upgrade_mode) {
       case TransportSecurityState::STSState::MODE_FORCE_HTTPS:
         serialized->SetString(kMode, kForceHTTPS);
         break;
       case TransportSecurityState::STSState::MODE_DEFAULT:
         serialized->SetString(kMode, kDefault);
         break;
       default:
         NOTREACHED() << "STSState with unknown mode";
         continue;
     }
 
-    toplevel.Set(key, std::move(serialized));
+    toplevel->Set(key, std::move(serialized));
   }
+}
 
-  TransportSecurityState::PKPStateIterator pkp_iterator(
-      *transport_security_state_);
+// Serializes PKP data from |state| into |toplevel|. For each PKP item in
+// |state|, if |toplevel| already contains an item for that hostname, the item
+// is updated with the PKP data.
+void SerializePKPData(TransportSecurityState* state,
+                      base::DictionaryValue* toplevel) {
+  base::Time now = base::Time::Now();
+  TransportSecurityState::PKPStateIterator pkp_iterator(*state);
   for (; pkp_iterator.HasNext(); pkp_iterator.Advance()) {
     const std::string& hostname = pkp_iterator.hostname();
     const TransportSecurityState::PKPState& pkp_state =
         pkp_iterator.domain_state();
 
     // See if the current |hostname| already has STS state and, if so, update
     // that entry.
     const std::string key = HashedDomainToExternalString(hostname);
     base::DictionaryValue* serialized = nullptr;
-    if (!toplevel.GetDictionary(key, &serialized)) {
+    if (!toplevel->GetDictionary(key, &serialized)) {
       std::unique_ptr<base::DictionaryValue> serialized_scoped(
           new base::DictionaryValue);
       serialized = serialized_scoped.get();
       PopulateEntryWithDefaults(serialized);
-      toplevel.Set(key, std::move(serialized_scoped));
+      toplevel->Set(key, std::move(serialized_scoped));
     }
 
     serialized->SetBoolean(kPkpIncludeSubdomains, pkp_state.include_subdomains);
     serialized->SetDouble(kPkpObserved, pkp_state.last_observed.ToDoubleT());
     serialized->SetDouble(kDynamicSPKIHashesExpiry,
                           pkp_state.expiry.ToDoubleT());
 
     // TODO(svaldez): Historically, both SHA-1 and SHA-256 hashes were
     // accepted in pins. Per spec, only SHA-256 is accepted now, however
     // existing serialized pins are still processed. Migrate historical pins
     // with SHA-1 hashes properly, either by dropping just the bad hashes or
     // the entire pin. See https://crbug.com/448501.
     if (now < pkp_state.expiry) {
       serialized->Set(kDynamicSPKIHashes,
                       SPKIHashesToListValue(pkp_state.spki_hashes));
     }
 
     serialized->SetString(kReportUri, pkp_state.report_uri.spec());
   }
+}
+
+// Serializes Expect-CT data from |state| into |toplevel|. For each Expect-CT
+// item in |state|, if |toplevel| already contains an item for that hostname,
+// the item is updated with the Expect-CT data.
+void SerializeExpectCTData(TransportSecurityState* state,
+                           base::DictionaryValue* toplevel) {
+  if (!IsDynamicExpectCTEnabled())
+    return;
+  TransportSecurityState::ExpectCTStateIterator expect_ct_iterator(*state);
+  for (; expect_ct_iterator.HasNext(); expect_ct_iterator.Advance()) {
+    const std::string& hostname = expect_ct_iterator.hostname();
+    const TransportSecurityState::ExpectCTState& expect_ct_state =
+        expect_ct_iterator.domain_state();
+
+    // See if the current |hostname| already has STS/PKP state and, if so,
+    // update that entry.
+    const std::string key = HashedDomainToExternalString(hostname);
+    base::DictionaryValue* serialized = nullptr;
+    if (!toplevel->GetDictionary(key, &serialized)) {
+      std::unique_ptr<base::DictionaryValue> serialized_scoped(
+          new base::DictionaryValue);
+      serialized = serialized_scoped.get();
+      PopulateEntryWithDefaults(serialized);
+      toplevel->Set(key, std::move(serialized_scoped));
+    }
+
+    serialized->SetDouble(kExpectCTObserved,
+                          expect_ct_state.last_observed.ToDoubleT());
+    serialized->SetDouble(kExpectCTExpiry, expect_ct_state.expiry.ToDoubleT());
+    serialized->SetBoolean(kExpectCTEnforce, expect_ct_state.enforce);
+    serialized->SetString(kExpectCTReportUri,
+                          expect_ct_state.report_uri.spec());
+  }
+}
+
+bool DeserializeExpectCTState(const base::DictionaryValue* parsed,
+                              TransportSecurityState::ExpectCTState* state) {
+  double observed;
+  bool has_observed = parsed->GetDouble(kExpectCTObserved, &observed);
+  double expiry;
+  bool has_expiry = parsed->GetDouble(kExpectCTExpiry, &expiry);
+  bool enforce;
+  bool has_enforce = parsed->GetBoolean(kExpectCTEnforce, &enforce);
+  std::string report_uri_str;
+  bool has_report_uri = parsed->GetString(kExpectCTReportUri, &report_uri_str);
+
+  // Entries are not required to have Expect-CT data.
+  if (!has_observed && !has_expiry && !has_enforce)

[mattm, 2017/04/15 04:45:01]

Is the PopulateEntryWithDefaults change necessary if you already check whether
the entries are present?

[estark, 2017/04/15 20:18:03]

Hmm no it's not, removed.

+    return true;
+
+  // If there is Expect-CT data, the default keys (kExpectCTObserved,
+  // kExpectCTExpiry, kExpectCTEnforce) are expected to be present.
+  if (!has_observed || !has_expiry || !has_enforce)
+    return false;
+
+  state->last_observed = base::Time::FromDoubleT(observed);
+  state->expiry = base::Time::FromDoubleT(expiry);
+  state->enforce = enforce;
+  if (has_report_uri) {
+    GURL report_uri(report_uri_str);
+    if (report_uri.is_valid())
+      state->report_uri = report_uri;
+  }
+  return true;
+}
+
+}  // namespace
+
+TransportSecurityPersister::TransportSecurityPersister(
+    TransportSecurityState* state,
+    const base::FilePath& profile_path,
+    const scoped_refptr<base::SequencedTaskRunner>& background_runner,
+    bool readonly)
+    : transport_security_state_(state),
+      writer_(profile_path.AppendASCII("TransportSecurity"), background_runner),
+      foreground_runner_(base::ThreadTaskRunnerHandle::Get()),
+      background_runner_(background_runner),
+      readonly_(readonly),
+      weak_ptr_factory_(this) {
+  transport_security_state_->SetDelegate(this);
+
+  base::PostTaskAndReplyWithResult(
+      background_runner_.get(), FROM_HERE,
+      base::Bind(&LoadState, writer_.path()),
+      base::Bind(&TransportSecurityPersister::CompleteLoad,
+                 weak_ptr_factory_.GetWeakPtr()));
+}
+
+TransportSecurityPersister::~TransportSecurityPersister() {
+  DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
+
+  if (writer_.HasPendingWrite())
+    writer_.DoScheduledWrite();
+
+  transport_security_state_->SetDelegate(NULL);
+}
+
+void TransportSecurityPersister::StateIsDirty(TransportSecurityState* state) {
+  DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
+  DCHECK_EQ(transport_security_state_, state);
+
+  if (!readonly_)
+    writer_.ScheduleWrite(this);
+}
+
+bool TransportSecurityPersister::SerializeData(std::string* output) {
+  DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
+
+  base::DictionaryValue toplevel;
+
+  // TODO(davidben): Fix the serialization format by splitting the on-disk
+  // representation of the STS and PKP states. https://crbug.com/470295.
+  SerializeSTSData(transport_security_state_, &toplevel);
+  SerializePKPData(transport_security_state_, &toplevel);
+  SerializeExpectCTData(transport_security_state_, &toplevel);
 
   base::JSONWriter::WriteWithOptions(
       toplevel, base::JSONWriter::OPTIONS_PRETTY_PRINT, output);
   return true;
 }
 
 bool TransportSecurityPersister::LoadEntries(const std::string& serialized,
                                              bool* dirty) {
   DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
 
@@ skipping 16 matching lines @@
   for (base::DictionaryValue::Iterator i(*dict_value);
        !i.IsAtEnd(); i.Advance()) {
     const base::DictionaryValue* parsed = NULL;
     if (!i.value().GetAsDictionary(&parsed)) {
       LOG(WARNING) << "Could not parse entry " << i.key() << "; skipping entry";
       continue;
     }
 
     TransportSecurityState::STSState sts_state;
     TransportSecurityState::PKPState pkp_state;
+    TransportSecurityState::ExpectCTState expect_ct_state;
 
     // kIncludeSubdomains is a legacy synonym for kStsIncludeSubdomains and
     // kPkpIncludeSubdomains. Parse at least one of these properties,
     // preferably the new ones.
     bool include_subdomains = false;
     bool parsed_include_subdomains = parsed->GetBoolean(kIncludeSubdomains,
                                                         &include_subdomains);
     sts_state.include_subdomains = include_subdomains;
     pkp_state.include_subdomains = include_subdomains;
     if (parsed->GetBoolean(kStsIncludeSubdomains, &include_subdomains)) {
@@ skipping 62 matching lines @@
     }
     if (parsed->GetDouble(kPkpObserved, &pkp_observed)) {
       pkp_state.last_observed = base::Time::FromDoubleT(pkp_observed);
     } else if (parsed->GetDouble(kCreated, &pkp_observed)) {
       pkp_state.last_observed = base::Time::FromDoubleT(pkp_observed);
     } else {
       dirtied = true;
       pkp_state.last_observed = base::Time::Now();
     }
 
+    if (!DeserializeExpectCTState(parsed, &expect_ct_state)) {
+      continue;
+    }
+
     bool has_sts =
         sts_state.expiry > current_time && sts_state.ShouldUpgradeToSSL();
     bool has_pkp =
         pkp_state.expiry > current_time && pkp_state.HasPublicKeyPins();
-    if (!has_sts && !has_pkp) {
+    bool has_expect_ct =
+        expect_ct_state.expiry > current_time &&
+        (expect_ct_state.enforce || !expect_ct_state.report_uri.is_empty());
+    if (!has_sts && !has_pkp && !has_expect_ct) {
       // Make sure we dirty the state if we drop an entry. The entries can only
-      // be dropped when both the STS and PKP states are expired or invalid.
+      // be dropped when all the STS, PKP, and Expect-CT states are expired or
+      // invalid.
       dirtied = true;
       continue;
     }
 
     std::string hashed = ExternalStringToHashedDomain(i.key());
     if (hashed.empty()) {
       dirtied = true;
       continue;
     }
 
     // Until the on-disk storage is split, there will always be 'null' entries.
     // We only register entries that have actual state.
     if (has_sts)
       state->AddOrUpdateEnabledSTSHosts(hashed, sts_state);
     if (has_pkp)
       state->AddOrUpdateEnabledPKPHosts(hashed, pkp_state);
+    if (has_expect_ct)
+      state->AddOrUpdateEnabledExpectCTHosts(hashed, expect_ct_state);
   }
 
   *dirty = dirtied;
   return true;
 }
 
-void TransportSecurityPersister::PopulateEntryWithDefaults(
-    base::DictionaryValue* host) {
-  host->Clear();
-
-  // STS default values.
-  host->SetBoolean(kStsIncludeSubdomains, false);
-  host->SetDouble(kStsObserved, 0.0);
-  host->SetDouble(kExpiry, 0.0);
-  host->SetString(kMode, kDefault);
-
-  // PKP default values.
-  host->SetBoolean(kPkpIncludeSubdomains, false);
-  host->SetDouble(kPkpObserved, 0.0);
-  host->SetDouble(kDynamicSPKIHashesExpiry, 0.0);
-}
-
 void TransportSecurityPersister::CompleteLoad(const std::string& state) {
   DCHECK(foreground_runner_->RunsTasksOnCurrentThread());
 
   if (state.empty())
     return;
 
   bool dirty = false;
   if (!LoadEntries(state, &dirty)) {
     LOG(ERROR) << "Failed to deserialize state: " << state;
     return;
   }
   if (dirty)
     StateIsDirty(transport_security_state_);
 }
 
 }  // namespace net