DNR Prototype: With flatbuffers

extensions/common/api/declarative_net_request/rules_indexer_util.cc

Index: extensions/common/api/declarative_net_request/rules_indexer_util.cc
diff --git a/extensions/common/api/declarative_net_request/rules_indexer_util.cc b/extensions/common/api/declarative_net_request/rules_indexer_util.cc
new file mode 100644
index 0000000000000000000000000000000000000000..bf13588233c0459de2de2c4c337a4eec692c4166
--- /dev/null
+++ b/extensions/common/api/declarative_net_request/rules_indexer_util.cc
@@ -0,0 +1,214 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "extensions/common/api/declarative_net_request/rules_indexer_util.h"
+
+#include <utility>
+
+#include <base/debug/stack_trace.h>
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/json/json_file_value_serializer.h"
+#include "base/metrics/histogram_macros.h"
+#include "base/stl_util.h"
+#include "base/strings/stringprintf.h"
+#include "base/threading/thread_restrictions.h"
+#include "extensions/common/api/declarative_net_request.h"
+#include "extensions/common/api/declarative_net_request/indexed_rule.h"
+#include "extensions/common/api/declarative_net_request/ruleset_indexer.h"
+
+namespace extensions {
+namespace declarative_net_request {
+
+namespace {
+
+struct ParseInfo {
+  ParseInfo(ParseResult parse_result, int index = -1)
+      : result(parse_result), rule_index(index) {}
+
+  ParseResult result;
+  // A value of -1 denotes an empty |rule_index|.
+  int rule_index;
+};
+
+bool PersistRuleset(const base::FilePath& path,
+                    const RulesetIndexer::SerializedData& data) {
+  return base::WriteFile(path, reinterpret_cast<const char*>(data.first),
+                         base::checked_cast<int>(data.second)) != -1;
+}
+
+// TODO use localized string?
+std::string GenerateErrorDescription(const ParseInfo& info) {
+  std::string error;
+  switch (info.result) {
+    case ParseResult::SUCCESS:
+      NOTREACHED();
+      break;
+    case ParseResult::ERROR_LIST_NOT_PASSED:
+      DCHECK_EQ(-1, info.rule_index);
+      error = "The ruleset file does not contain a valid list";
+      break;
+    case ParseResult::ERROR_DUPLICATE_IDS:
+      DCHECK_EQ(-1, info.rule_index);
+      error = "Rules with duplicate ids found.";
+      break;
+    case ParseResult::ERROR_RESOURCE_TYPE_DUPLICATED:
+      DCHECK_GE(info.rule_index, 0);
+      error = base::StringPrintf(
+          "Rule at index %d has duplicated resource type between %s and %s "
+          "keys",
+          info.rule_index, kResourceTypesKey, kExcludeResourceTypesKey);
+      break;
+    case ParseResult::ERROR_PERSISTING_RULESET:
+      DCHECK_EQ(-1, info.rule_index);
+      error = "The rules file could not be persisted";
+      break;
+    // case ParseResult::ERROR_EMPTY_RULE_ID:
+    //   DCHECK_GE(info.rule_index, 0);
+    //   error = base::StringPrintf(
+    //       "Rule at index %d does not have a valid value for %s key",
+    //       info.rule_index, kRuleIDKey);
+    //   break;
+    case ParseResult::ERROR_EMPTY_REDIRECT_RULE_PRIORITY:
+      DCHECK_GE(info.rule_index, 0);
+      error = base::StringPrintf(
+          "Rule at index %d does not have a valid value for %s key. This is "
+          "required for redirect rules",
+          info.rule_index, kRulePriorityKey);
+      break;
+    case ParseResult::ERROR_EMPTY_REDIRECT_URL:
+      DCHECK_GE(info.rule_index, 0);
+      error = base::StringPrintf(
+          "Rule at index %d does not have a valid value for %s.%s key. This is "
+          "required for redirect rules",
+          info.rule_index, kRuleActionKey, kRedirectUrlKey);
+      break;
+    case ParseResult::ERROR_INVALID_RULE_ID:
+      DCHECK_GE(info.rule_index, 0);
+      error = base::StringPrintf(
+          "Rule at index %d has an invalid value for %s key. This should be "
+          "greater than or equal to %d",
+          info.rule_index, kRuleIDKey, kMinValidID);
+      break;
+    case ParseResult::ERROR_INVALID_REDIRECT_RULE_PRIORITY:
+      DCHECK_GE(info.rule_index, 0);
+      error = base::StringPrintf(
+          "Rule at index %d has an invalid value for %s key. This should be "
+          "greater than or equal to %d",
+          info.rule_index, kRulePriorityKey, kMinValidPriority);
+      break;
+    case ParseResult::ERROR_JSON_PARSE:
+      DCHECK_GE(info.rule_index, 0);
+      error = base::StringPrintf("Rule at index %d could not be parsed.",
+                                 info.rule_index);
+      break;
+  }
+  return base::StringPrintf("%s: %s", kAPIName, error.c_str());
+}
+
+ParseInfo IndexAndPersistRuleset(const base::Value& rules,
+                                 const base::FilePath& indexed_ruleset_path) {
+  base::ThreadRestrictions::AssertIOAllowed();
+
+  const base::ListValue* rules_list = nullptr;
+  if (!rules.GetAsList(&rules_list))
+    return ParseInfo(ParseResult::ERROR_LIST_NOT_PASSED);
+
+  RulesetIndexer indexer;
+  std::set<int> id_set;  // Ensure all ids are distinct.
+  base::TimeTicks start_json_to_flatbuffer = base::TimeTicks::Now();
+  base::TimeDelta create_parsed_rule;
+  base::TimeDelta add_to_flatbuffer;
+  base::TimeDelta create_indexed_rule;
+  std::unique_ptr<Rule> parsed_rule;
+  for (auto iter = rules_list->begin(); iter != rules_list->end(); iter++) {
+    const size_t index = iter - rules_list->begin();
+
+    ParseResult parse_result = ParseResult::SUCCESS;
+    {
+      base::TimeTicks start = base::TimeTicks::Now();
+      parsed_rule = Rule::FromValue(*iter);
+      if (!parsed_rule)
+        parse_result = ParseResult::ERROR_JSON_PARSE;
+      create_parsed_rule += base::TimeTicks::Now() - start;
+    }
+
+    if (parse_result != ParseResult::SUCCESS)
+      return ParseInfo(parse_result, index);
+
+    if (base::ContainsKey(id_set, parsed_rule->id))
+      return ParseInfo(ParseResult::ERROR_DUPLICATE_IDS, index);
+
+    IndexedRule indexed_rule;
+    {
+      base::TimeTicks start = base::TimeTicks::Now();
+      parse_result = CreateIndexedRule(std::move(parsed_rule), &indexed_rule);
+      create_indexed_rule += base::TimeTicks::Now() - start;
+    }
+
+    if (parse_result != ParseResult::SUCCESS)
+      return ParseInfo(parse_result, index);
+
+    {
+      base::TimeTicks start = base::TimeTicks::Now();
+      indexer.AddUrlRule(indexed_rule);
+      add_to_flatbuffer += base::TimeTicks::Now() - start;
+    }
+  }
+
+  LOG(ERROR) << "--------create_parsed_rule " << create_parsed_rule << "\n";
+  LOG(ERROR) << "--------create_indexed_rule " << create_indexed_rule << "\n";
+  LOG(ERROR) << "--------add_to_flatbuffer " << add_to_flatbuffer << "\n";
+  base::TimeDelta end_json_to_flatbuffer =
+      base::TimeTicks::Now() - start_json_to_flatbuffer;
+  LOG(ERROR) << "--------end_json_to_flatbuffer " << end_json_to_flatbuffer
+             << "\n";
+
+  // The actual data buffer is still owned by |indexer|.
+  RulesetIndexer::SerializedData data = indexer.FinishAndGetData();
+
+  base::TimeTicks start_persist = base::TimeTicks::Now();
+  if (!PersistRuleset(indexed_ruleset_path, data))
+    return ParseInfo(ParseResult::ERROR_PERSISTING_RULESET);
+  base::TimeDelta end_persist = base::TimeTicks::Now() - start_persist;
+  LOG(ERROR) << "--------end_persist " << end_persist << "\n";
+
+  return ParseInfo(ParseResult::SUCCESS);
+}
+
+}  // namespace
+
+bool IndexAndPersistRuleset(const base::FilePath& json_rules_path,
+                            const base::FilePath& indexed_ruleset_path,
+                            std::string* error) {
+  base::TimeTicks start_index_and_persist = base::TimeTicks::Now();
+  base::ThreadRestrictions::AssertIOAllowed();
+
+  base::TimeTicks start_deserialization = base::TimeTicks::Now();
+  JSONFileValueDeserializer deserializer(json_rules_path);
+  std::unique_ptr<base::Value> rules = deserializer.Deserialize(nullptr, error);
+  if (!rules)
+    return false;  // |error| will be populated by the call to Deserialize.
+  base::TimeDelta deserialize_time =
+      base::TimeTicks::Now() - start_deserialization;
+  LOG(ERROR) << "--------deserialize_time " << deserialize_time << "\n";
+  UMA_HISTOGRAM_TIMES("DNR.JSONDeserializeTime", deserialize_time);
+
+  const ParseInfo info = IndexAndPersistRuleset(*rules, indexed_ruleset_path);
+
+  base::TimeDelta index_and_persist_time =
+      base::TimeTicks::Now() - start_index_and_persist;
+  LOG(ERROR) << "--------index_and_persist_time " << index_and_persist_time
+             << "\n";
+  UMA_HISTOGRAM_TIMES("DNR.IndexAndPersistRuleset", index_and_persist_time);
+  if (info.result != ParseResult::SUCCESS) {
+    if (error)
+      *error = GenerateErrorDescription(info);
+    return false;
+  }
+  return true;
+}
+
+}  // namespace declarative_net_request
+}  // namespace extensions