Add integration browser tests for settings hardening.

chrome/browser/prefs/tracked/pref_hash_browsertest.cc

Index: chrome/browser/prefs/tracked/pref_hash_browsertest.cc
diff --git a/chrome/browser/prefs/tracked/pref_hash_browsertest.cc b/chrome/browser/prefs/tracked/pref_hash_browsertest.cc
new file mode 100644
index 0000000000000000000000000000000000000000..5adc110e4ad7c93e8ee2460607db4eddebef6daf
--- /dev/null
+++ b/chrome/browser/prefs/tracked/pref_hash_browsertest.cc
@@ -0,0 +1,500 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+
+#include "base/command_line.h"
+#include "base/file_util.h"
+#include "base/files/file_path.h"
+#include "base/json/json_file_value_serializer.h"
+#include "base/metrics/histogram_base.h"
+#include "base/metrics/histogram_samples.h"
+#include "base/metrics/statistics_recorder.h"
+#include "base/path_service.h"
+#include "base/prefs/pref_service.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_util.h"
+#include "base/values.h"
+#include "build/build_config.h"
+#include "chrome/browser/extensions/extension_browsertest.h"
+#include "chrome/browser/extensions/extension_service.h"
+#include "chrome/browser/prefs/chrome_pref_service_factory.h"
+#include "chrome/browser/prefs/profile_pref_store_manager.h"
+#include "chrome/browser/prefs/session_startup_pref.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/common/chrome_constants.h"
+#include "chrome/common/chrome_paths.h"
+#include "chrome/common/pref_names.h"
+#include "chrome/test/base/testing_profile.h"
+#include "components/search_engines/default_search_manager.h"
+#include "content/public/common/content_switches.h"
+
+#if defined(OS_CHROMEOS)
+#include "chromeos/chromeos_switches.h"
+#endif
+
+namespace {
+
+// Extension ID of chrome/test/data/extensions/good.crx
+const char kGoodCrxId[] = "ldnnhddmnhbkjipkidpdiheffobcpfmf";
+
+// Returns the number of times |histogram_name| was reported so far; adding the
+// results of the first 100 buckets (there are only ~18 reporting IDs as of this
+// writing; varies depending on the platform). If |expect_zero| is true, this
+// method will explicitly report IDs that are non-zero for ease of diagnosis.
+int GetTrackedPrefHistogramCount(const char* histogram_name, bool expect_zero) {
+  const base::HistogramBase* histogram =
+      base::StatisticsRecorder::FindHistogram(histogram_name);
+  if (!histogram)
+    return 0;
+
+  scoped_ptr<base::HistogramSamples> samples(histogram->SnapshotSamples());
+  int sum = 0;
+  for (int i = 0; i < 18; ++i) {
+    int count_for_id = samples->GetCount(i);
+    sum += count_for_id;
+
+    //FIXME needed?!
+    // FIXME use size_t for return value here?
+    if (expect_zero)
+      EXPECT_EQ(0, count_for_id) << "Faulty reporting_id: " << i;

[gab, 2014/07/31 13:19:14]

Please ignore this for now.

I'm using it for debugging, but I'll most likely change how this works before
the last patch set.

+  }
+  return sum;
+}
+
+base::DictionaryValue* ReadPrefsDictionary(
+    JSONFileValueSerializer* serializer) {
+  int error_code = JSONFileValueSerializer::JSON_NO_ERROR;
+  std::string error_str;
+  base::Value* prefs = serializer->Deserialize(&error_code, &error_str);
+  if (!prefs || error_code != JSONFileValueSerializer::JSON_NO_ERROR) {
+    ADD_FAILURE() << "Error #" << error_code << ": " << error_str;
+    return NULL;
+  }
+  if (!prefs->IsType(base::Value::TYPE_DICTIONARY)) {
+    ADD_FAILURE();
+    return NULL;
+  }
+  return static_cast<base::DictionaryValue*>(prefs);
+}
+
+#define PREF_HASH_BROWSER_TEST(fixture, test_name)                          \
+  IN_PROC_BROWSER_TEST_P(fixture, PRE_##test_name) {                        \
+    SetupPreferences();                                                     \
+  }                                                                         \
+  IN_PROC_BROWSER_TEST_P(fixture, test_name) {                              \
+    VerifyReactionToPrefAttack();                                           \
+  }                                                                         \
+  INSTANTIATE_TEST_CASE_P(                                                  \
+      fixture##Instance,                                                    \
+      fixture,                                                              \
+      testing::Values(                                                      \
+          chrome_prefs::internals::kSettingsEnforcementGroupNoEnforcement,  \
+          chrome_prefs::internals::kSettingsEnforcementGroupEnforceAlways,  \
+          chrome_prefs::internals::                                         \
+              kSettingsEnforcementGroupEnforceAlwaysWithDSE,                \
+          chrome_prefs::internals::                                         \
+              kSettingsEnforcementGroupEnforceAlwaysWithExtensionsAndDSE));
+
+// A base fixture designed such that implementations do two things:
+//  1) Override all three pure-virtual methods below to setup, attack, and
+//     verify preferenes throughout the tests provided by this fixture.
+//  2) Instantiate their test via the PREF_HASH_BROWSER_TEST macro above.
+// Based on top of ExtensionBrowserTest to allow easy interaction with the
+// ExtensionService.
+class PrefHashBrowserTestBase
+    : public ExtensionBrowserTest,
+      public testing::WithParamInterface<std::string> {
+ public:
+  // List of potential protection levels for this test in strict increasing
+  // order of protection levels.
+  enum SettingsProtectionLevel {
+    PROTECTION_DISABLED_ON_PLATFORM,
+    PROTECTION_DISABLED_FOR_GROUP,
+    PROTECTION_ENABLED_BASIC,
+    PROTECTION_ENABLED_DSE,
+    PROTECTION_ENABLED_EXTENSIONS,
+  };
+
+  PrefHashBrowserTestBase()
+      : protection_level_(PROTECTION_DISABLED_ON_PLATFORM) {
+    if (ProfilePrefStoreManager::kPlatformSupportsPreferenceTracking) {
+      const std::string& experiment_group = GetParam();
+      if (experiment_group ==
+          chrome_prefs::internals::kSettingsEnforcementGroupNoEnforcement) {
+        protection_level_ = PROTECTION_DISABLED_FOR_GROUP;
+      } else if (experiment_group ==
+                 chrome_prefs::internals::
+                     kSettingsEnforcementGroupEnforceAlways) {
+        protection_level_ = PROTECTION_ENABLED_BASIC;
+      } else if (experiment_group ==
+                 chrome_prefs::internals::
+                     kSettingsEnforcementGroupEnforceAlwaysWithDSE) {
+        protection_level_ = PROTECTION_ENABLED_DSE;
+      } else if (experiment_group ==
+                 chrome_prefs::internals::
+                     kSettingsEnforcementGroupEnforceAlwaysWithExtensionsAndDSE) {
+        protection_level_ = PROTECTION_ENABLED_EXTENSIONS;
+      } else {
+        ADD_FAILURE();
+      }
+    }
+  }
+
+  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
+    ExtensionBrowserTest::SetUpCommandLine(command_line);
+    EXPECT_FALSE(command_line->HasSwitch(switches::kForceFieldTrials));
+    command_line->AppendSwitchASCII(
+        switches::kForceFieldTrials,
+        std::string(chrome_prefs::internals::kSettingsEnforcementTrialName) +
+            "/" + GetParam() + "/");
+#if defined(OS_CHROMEOS)
+    command_line->AppendSwitch(
+        chromeos::switches::kIgnoreUserProfileMappingForTests);
+#endif
+  }
+
+  virtual bool SetUpUserDataDirectory() OVERRIDE {
+    // Do the normal setup in the PRE test and attack preferences in the main
+    // test.
+    if (IsPRETest())
+      return ExtensionBrowserTest::SetUpUserDataDirectory();
+
+    base::FilePath profile_dir;
+    EXPECT_TRUE(PathService::Get(chrome::DIR_USER_DATA, &profile_dir));
+    profile_dir = profile_dir.AppendASCII(TestingProfile::kTestUserProfileDir);
+
+    AttackPreferencesOnDisk(profile_dir);
+
+    return true;
+  }
+
+  // In the PRE_ test, find the number of tracked preferences that were
+  // initialized and save it to a file to be read back in the main test and used
+  // as the total number of tracked preferences.
+  virtual void SetUpOnMainThread() OVERRIDE {
+    ExtensionBrowserTest::SetUpOnMainThread();
+
+    // File in which the PRE_ test will save the number of tracked preferences
+    // on this platform.
+    const char kNumTrackedPrefFilename[] = "NumTrackedPrefs";
+
+    base::FilePath num_tracked_prefs_file;
+    ASSERT_TRUE(
+        PathService::Get(chrome::DIR_USER_DATA, &num_tracked_prefs_file));
+    num_tracked_prefs_file =
+        num_tracked_prefs_file.AppendASCII(kNumTrackedPrefFilename);
+
+    if (IsPRETest()) {
+      num_tracked_prefs_ = GetTrackedPrefHistogramCount(
+          "Settings.TrackedPreferenceTrustedInitialized", false);
+      EXPECT_EQ(protection_level_ > PROTECTION_DISABLED_ON_PLATFORM,
+                num_tracked_prefs_ > 0);
+
+      // Split tracked prefs are reported as Unchanged not as TrustedInitialized
+      // when an empty dictionary is encountered on first run.
+      int num_split_tracked_prefs = GetTrackedPrefHistogramCount(
+          "Settings.TrackedPreferenceUnchanged", false);
+      EXPECT_EQ(protection_level_ > PROTECTION_DISABLED_ON_PLATFORM ? 1 : 0,
+                num_split_tracked_prefs);
+
+      num_tracked_prefs_ += num_split_tracked_prefs;
+
+      std::string num_tracked_prefs_str = base::IntToString(num_tracked_prefs_);
+      EXPECT_EQ(num_tracked_prefs_str.size(),
+                base::WriteFile(num_tracked_prefs_file,
+                                num_tracked_prefs_str.c_str(),
+                                num_tracked_prefs_str.size()));
+    } else {
+      std::string num_tracked_prefs_str;
+      EXPECT_TRUE(base::ReadFileToString(num_tracked_prefs_file,
+                                         &num_tracked_prefs_str));
+      EXPECT_TRUE(
+          base::StringToInt(num_tracked_prefs_str, &num_tracked_prefs_));
+    }
+  }
+
+ protected:
+  // Called from the PRE_ test's body. Overrides should use it to setup
+  // preferences through Chrome.
+  virtual void SetupPreferences() = 0;
+
+  // Called prior to the main test launching its browser. Overrides should use
+  // it to attack preferences in |profile_dir| on disk.
+  virtual void AttackPreferencesOnDisk(const base::FilePath& profile_dir) = 0;
+
+  // Called from the body of the main test. Overrides should use it to verify
+  // that the browser had the desired reaction when faced when the attack
+  // orchestrated in AttackPreferencesOnDisk().
+  virtual void VerifyReactionToPrefAttack() = 0;
+
+  int num_tracked_prefs_;
+
+  SettingsProtectionLevel protection_level_;
+
+ private:
+  // Returns true if this is the PRE_ phase of the test.
+  bool IsPRETest() {
+    return StartsWithASCII(
+        testing::UnitTest::GetInstance()->current_test_info()->name(),
+        "PRE_",
+        true  /* case_sensitive */);
+  }
+};
+
+}  // namespace
+
+// Verifies that nothing is reset when nothing is tampered with.
+// Also sanity checks that the expected preferences files are in place.
+class PrefHashBrowserTestUnchangedDefault : public PrefHashBrowserTestBase {
+ public:
+  virtual void SetupPreferences() OVERRIDE {
+    // Default Chrome setup.
+  }
+
+  virtual void AttackPreferencesOnDisk(
+      const base::FilePath& profile_dir) OVERRIDE {
+    // No attack, just sanity check pref setup on disk.
+    EXPECT_TRUE(
+        base::PathExists(profile_dir.Append(chrome::kPreferencesFilename)));
+    EXPECT_FALSE(base::PathExists(
+        profile_dir.Append(chrome::kProtectedPreferencesFilenameDeprecated)));
+    EXPECT_TRUE(base::PathExists(
+        profile_dir.Append(chrome::kSecurePreferencesFilename)));
+  }
+
+  virtual void VerifyReactionToPrefAttack() OVERRIDE {
+    // Expect all prefs to be reported as Unchanged with no resets.
+    EXPECT_EQ(num_tracked_prefs_,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceUnchanged", false));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceWantedReset", true));
+    EXPECT_EQ(
+        0,
+        GetTrackedPrefHistogramCount("Settings.TrackedPreferenceReset", true));
+
+    // Nothing else should have triggered.
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount("Settings.TrackedPreferenceChanged",
+                                           true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount("Settings.TrackedPreferenceCleared",
+                                           true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceInitialized", true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceTrustedInitialized", true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceMigratedLegacyDeviceId", true));
+  }
+};
+
+PREF_HASH_BROWSER_TEST(PrefHashBrowserTestUnchangedDefault, UnchangedDefault);
+
+// Augments PrefHashBrowserTestUnchangedDefault to confirm that nothing is reset
+// when nothing is tampered with, even if Chrome itself wrote custom prefs in
+// its last run.
+class PrefHashBrowserTestUnchangedCustom
+    : public PrefHashBrowserTestUnchangedDefault {
+ public:
+  virtual void SetupPreferences() OVERRIDE {
+    profile()->GetPrefs()->SetString(prefs::kHomePage, "http://example.com");
+
+    InstallExtensionWithUIAutoConfirm(
+        test_data_dir_.AppendASCII("good.crx"), 1, browser());
+  }
+
+  virtual void VerifyReactionToPrefAttack() OVERRIDE {
+    // Make sure the settings written in the last run stuck.
+    EXPECT_EQ("http://example.com",
+              profile()->GetPrefs()->GetString(prefs::kHomePage));
+
+    EXPECT_TRUE(extension_service()->GetExtensionById(kGoodCrxId, false));
+
+    // Reaction should be identical to unattacked default prefs.
+    PrefHashBrowserTestUnchangedDefault::VerifyReactionToPrefAttack();
+  }
+};
+
+PREF_HASH_BROWSER_TEST(PrefHashBrowserTestUnchangedCustom, UnchangedCustom);
+
+// Verifies that cleared prefs are reported.
+class PrefHashBrowserTestClearedAtomic : public PrefHashBrowserTestBase {
+ public:
+  virtual void SetupPreferences() OVERRIDE {
+    profile()->GetPrefs()->SetString(prefs::kHomePage, "http://example.com");
+  }
+
+  virtual void AttackPreferencesOnDisk(
+      const base::FilePath& profile_dir) OVERRIDE {
+    JSONFileValueSerializer serializer(

[erikwright (departed), 2014/07/31 14:38:26]

Presumably this stanza will appear frequently.

What if the caller of this method passed in two
scoped_ptr<base::DictionaryValue>* for normal and secure prefs?

You could edit them, clear them, or replace them, and then the caller would
write them to disk.

[gab, 2014/07/31 19:14:16]

Great idea, done (didn't use scoped_ptr*'s in the AttackPreferencesOnDisk() call
though as I don't intend to go as far as deleting/creating the file if it DNE
yet).

+        profile_dir.Append(protection_level_ >= PROTECTION_ENABLED_BASIC
+                               ? chrome::kSecurePreferencesFilename
+                               : chrome::kPreferencesFilename));
+
+    base::DictionaryValue* prefs = ReadPrefsDictionary(&serializer);
+    EXPECT_TRUE(prefs->Remove(prefs::kHomePage, NULL));
+    EXPECT_TRUE(serializer.Serialize(*prefs));
+  }
+
+  virtual void VerifyReactionToPrefAttack() OVERRIDE {
+    // The clearance of homepage should have been noticed, but shouldn't have
+    // triggered a reset (as there is nothing we can do when the pref is already
+    // gone).
+    EXPECT_EQ(1,
+              GetTrackedPrefHistogramCount("Settings.TrackedPreferenceCleared",
+                                           false));
+    EXPECT_EQ(num_tracked_prefs_ - 1,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceUnchanged", false));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceWantedReset", true));
+    EXPECT_EQ(
+        0,
+        GetTrackedPrefHistogramCount("Settings.TrackedPreferenceReset", true));
+
+    // Nothing else should have triggered.
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount("Settings.TrackedPreferenceChanged",
+                                           true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceInitialized", true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceTrustedInitialized", true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceMigratedLegacyDeviceId", true));
+  }
+};
+
+PREF_HASH_BROWSER_TEST(PrefHashBrowserTestClearedAtomic, ClearedAtomic);
+
+// Verifies that clearing the MACs results in untrusted Initialized pings for
+// non-null protected prefs.
+class PrefHashBrowserTestUntrustedInitialized : public PrefHashBrowserTestBase {
+ public:
+  virtual void SetupPreferences() OVERRIDE {
+    // Explicitly set the DSE (it's otherwise NULL by default, preventing
+    // thorough testing of the PROTECTION_ENABLED_DSE level).
+    DefaultSearchManager default_search_manager(
+        profile()->GetPrefs(), DefaultSearchManager::ObserverCallback());
+    DefaultSearchManager::Source dse_source =
+        static_cast<DefaultSearchManager::Source>(-1);
+
+    const TemplateURLData* default_template_url_data =
+        default_search_manager.GetDefaultSearchEngine(&dse_source);
+    EXPECT_EQ(DefaultSearchManager::FROM_FALLBACK, dse_source);
+
+    default_search_manager.SetUserSelectedDefaultSearchEngine(
+        *default_template_url_data);
+
+    default_search_manager.GetDefaultSearchEngine(&dse_source);
+    EXPECT_EQ(DefaultSearchManager::FROM_USER, dse_source);
+
+    // Also explicitly set an atomic pref that falls under
+    // PROTECTION_ENABLED_BASIC.
+    profile()->GetPrefs()->SetInteger(prefs::kRestoreOnStartup,
+                                      SessionStartupPref::URLS);
+  }
+
+  virtual void AttackPreferencesOnDisk(
+      const base::FilePath& profile_dir) OVERRIDE {
+    static const base::FilePath::CharType* kPrefFiles[] = {
+        chrome::kPreferencesFilename, chrome::kSecurePreferencesFilename};
+    for (size_t i = 0; i < arraysize(kPrefFiles); ++i) {
+      JSONFileValueSerializer serializer(profile_dir.Append(kPrefFiles[i]));
+      base::DictionaryValue* prefs = ReadPrefsDictionary(&serializer);
+      EXPECT_TRUE(prefs->Remove("protection.macs", NULL));
+      EXPECT_TRUE(serializer.Serialize(*prefs));
+    }
+  }
+
+  virtual void VerifyReactionToPrefAttack() OVERRIDE {
+    // Preferences that are NULL by default will be TrustedInitialized.
+    int num_null_values = GetTrackedPrefHistogramCount(
+        "Settings.TrackedPreferenceTrustedInitialized", false);
+    EXPECT_EQ(protection_level_ > PROTECTION_DISABLED_ON_PLATFORM,
+              num_null_values > 0);
+    if (num_null_values > 0) {
+      // This test requires that at least 3 prefs be non-null (extensions, DSE,
+      // and 1 atomic pref explictly set for this test above).
+      EXPECT_LT(num_null_values, num_tracked_prefs_ - 3);
+    }
+
+    // Expect all non-null prefs to be reported as Initialized (with
+    // accompanying resets or wanted resets based on the current protection
+    // level).
+    EXPECT_EQ(num_tracked_prefs_ - num_null_values,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceInitialized", false));
+
+    int num_protected_prefs = 0;
+    // A switch statement falling through each protection level in decreasing
+    // levels of protection to add expectations for each level which augments
+    // the previous one.
+    switch (protection_level_) {
+      case PROTECTION_ENABLED_EXTENSIONS:
+        ++num_protected_prefs;
+        // Falls through.
+      case PROTECTION_ENABLED_DSE:
+        ++num_protected_prefs;
+        // Falls through.
+      case PROTECTION_ENABLED_BASIC:
+        num_protected_prefs += num_tracked_prefs_ - num_null_values - 2;
+        // Falls through.
+      case PROTECTION_DISABLED_FOR_GROUP:
+        // No protection. Falls through.
+      case PROTECTION_DISABLED_ON_PLATFORM:
+        // No protection.
+        break;
+    }
+
+    EXPECT_EQ(num_tracked_prefs_ - num_null_values - num_protected_prefs,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceWantedReset", false));
+    EXPECT_EQ(
+        num_protected_prefs,
+        GetTrackedPrefHistogramCount("Settings.TrackedPreferenceReset", false));
+
+    DefaultSearchManager default_search_manager(
+        profile()->GetPrefs(), DefaultSearchManager::ObserverCallback());
+    DefaultSearchManager::Source dse_source =
+        static_cast<DefaultSearchManager::Source>(-1);
+    default_search_manager.GetDefaultSearchEngine(&dse_source);
+    EXPECT_EQ(protection_level_ < PROTECTION_ENABLED_DSE
+                  ? DefaultSearchManager::FROM_USER
+                  : DefaultSearchManager::FROM_FALLBACK,
+              dse_source);
+
+    EXPECT_EQ(protection_level_ < PROTECTION_ENABLED_BASIC,
+              profile()->GetPrefs()->GetInteger(prefs::kRestoreOnStartup) ==
+                  SessionStartupPref::URLS);
+
+    // Nothing else should have triggered.
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceUnchanged", true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount("Settings.TrackedPreferenceChanged",
+                                           true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount("Settings.TrackedPreferenceCleared",
+                                           true));
+    EXPECT_EQ(0,
+              GetTrackedPrefHistogramCount(
+                  "Settings.TrackedPreferenceMigratedLegacyDeviceId", true));
+  }
+};
+
+PREF_HASH_BROWSER_TEST(PrefHashBrowserTestUntrustedInitialized,
+                       UntrustedInitialized);