Introduce a Doodle Fetcher for NTP

components/doodle/doodle_fetcher.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/doodle/doodle_fetcher.h"
+
+#include <utility>
+
+#include "base/strings/string_number_conversions.h"
+#include "base/time/clock.h"
+#include "base/time/default_clock.h"
+#include "base/time/time.h"
+#include "base/values.h"
+#include "components/google/core/browser/google_url_tracker.h"
+#include "components/google/core/browser/google_util.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_status_code.h"
+#include "net/url_request/url_fetcher.h"
+#include "url/gurl.h"
+
+using net::URLFetcher;
+
+namespace doodle {
+
+const char kDoodleConfigPath[] = "/async/ddljson";
+
+namespace {
+
+std::string StripSafetyPreamble(const std::string& json) {
+  // The response may start with )]}'. Ignore this.
+  const char kResponsePreamble[] = ")]}'";
+
+  base::StringPiece json_sp(json);
+  if (json_sp.starts_with(kResponsePreamble)) {
+    json_sp.remove_prefix(strlen(kResponsePreamble));
+  }
+
+  return json_sp.as_string();
+}
+
+DoodleType ParseDoodleType(const base::DictionaryValue& ddljson) {
+  std::string type_str;
+  ddljson.GetString("doodle_type", &type_str);
+  if (type_str == "SIMPLE") {
+    return DoodleType::SIMPLE;
+  }
+  if (type_str == "RANDOM") {
+    return DoodleType::RANDOM;
+  }
+  if (type_str == "VIDEO") {
+    return DoodleType::VIDEO;
+  }
+  if (type_str == "INTERACTIVE") {
+    return DoodleType::INTERACTIVE;
+  }
+  if (type_str == "INLINE_INTERACTIVE") {
+    return DoodleType::INLINE_INTERACTIVE;
+  }
+  if (type_str == "SLIDESHOW") {
+    return DoodleType::SLIDESHOW;
+  }
+  return DoodleType::UNKNOWN;
+}
+
+}  // namespace
+
+DoodleImage::DoodleImage()
+    : height(0), width(0), is_animated_gif(false), is_cta(false) {}
+DoodleImage::~DoodleImage() = default;
+
+DoodleConfig::DoodleConfig() : doodle_type(DoodleType::UNKNOWN) {}
+DoodleConfig::DoodleConfig(const DoodleConfig& config) = default;
+DoodleConfig::~DoodleConfig() = default;
+
+DoodleFetcher::DoodleFetcher(net::URLRequestContextGetter* download_context,
+                             GoogleURLTracker* google_url_tracker,
+                             const ParseJSONCallback& json_parsing_callback)
+    : download_context_(download_context),
+      json_parsing_callback_(json_parsing_callback),
+      google_url_tracker_(google_url_tracker),
+      clock_(new base::DefaultClock()),
+      weak_ptr_factory_(this) {}
+
+DoodleFetcher::~DoodleFetcher() {}
+
+void DoodleFetcher::FetchDoodle(FinishedCallback callback) {
+  callbacks_.push_back(std::move(callback));
+  if (callbacks_.size() > 1) {
+    // If there was already another callback, a request is still running.
+    return;
+  }
+  DCHECK(!fetcher_.get());
+  fetcher_ = URLFetcher::Create(GetGoogleBaseUrl().Resolve(kDoodleConfigPath),
+                                URLFetcher::GET, this);
+  fetcher_->SetRequestContext(download_context_);
+  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
+                         net::LOAD_DO_NOT_SAVE_COOKIES);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(1);
+  fetcher_->Start();
+}
+
+// net::URLFetcherDelegate implementation.
+void DoodleFetcher::OnURLFetchComplete(const net::URLFetcher* source) {
+  DCHECK_EQ(fetcher_.get(), source);
+  std::unique_ptr<net::URLFetcher> free_fetcher = std::move(fetcher_);
+
+  std::string json_string;
+  if (!(source->GetStatus().is_success() &&
+        source->GetResponseCode() == net::HTTP_OK &&
+        source->GetResponseAsString(&json_string))) {
+    RespondToAllCallbacks(DoodleState::DOWNLOAD_ERROR, base::nullopt);
+    return;
+  }
+
+  json_parsing_callback_.Run(
+      StripSafetyPreamble(std::move(json_string)),
+      base::Bind(&DoodleFetcher::OnJsonParsed, weak_ptr_factory_.GetWeakPtr()),
+      base::Bind(&DoodleFetcher::OnJsonParseFailed,
+                 weak_ptr_factory_.GetWeakPtr()));
+}
+
+void DoodleFetcher::OnJsonParsed(std::unique_ptr<base::Value> json) {
+  std::unique_ptr<base::DictionaryValue> dict =
+      base::DictionaryValue::From(std::move(json));
+  if (!dict.get()) {
+    LOG(ERROR) << "Doodle JSON is not valid";
+    RespondToAllCallbacks(DoodleState::PARSING_ERROR, base::nullopt);
+    return;
+  }
+
+  ParseDoodle(std::move(dict));
+}
+
+void DoodleFetcher::OnJsonParseFailed(const std::string& error_message) {
+  LOG(ERROR) << "JSON parsing failed: " << error_message;
+  RespondToAllCallbacks(DoodleState::PARSING_ERROR, base::nullopt);
+}
+
+void DoodleFetcher::ParseDoodle(std::unique_ptr<base::DictionaryValue> config) {
+  const base::DictionaryValue* ddljson = nullptr;
+  if (!config->GetDictionary("ddljson", &ddljson)) {
+    LOG(ERROR) << "Doodle JSON reponse did not contain 'ddljson' key.";
+    RespondToAllCallbacks(DoodleState::PARSING_ERROR, base::nullopt);
+    return;
+  }
+
+  DoodleConfig result;
+
+  ParseBaseInformation(*ddljson, &result);
+  if (!ParseImage(*ddljson, "large_image", &result.large_image)) {
+    RespondToAllCallbacks(DoodleState::NO_DOODLE, base::nullopt);
+    return;
+  }
+  ParseImage(*ddljson, "transparent_large_image",
+             &result.transparent_large_image);
+  ParseImage(*ddljson, "large_cta_image", &result.large_cta_image);
+
+  RespondToAllCallbacks(DoodleState::AVAILABLE, std::move(result));
+}
+
+bool DoodleFetcher::ParseImage(const base::DictionaryValue& image_parent,
+                               const std::string& image_name,
+                               DoodleImage* image) const {
+  DCHECK(image);
+  const base::DictionaryValue* image_dict = nullptr;
+  if (!image_parent.GetDictionary(image_name, &image_dict)) {
+    return false;
+  }
+  image->url = ParseRelativeUrl(*image_dict, "url");
+  if (!image->url.is_valid()) {
+    return false;
+  }
+  image_dict->GetInteger("height", &image->height);
+  image_dict->GetInteger("width", &image->width);
+  image_dict->GetBoolean("is_animated_gif", &image->is_animated_gif);
+  image_dict->GetBoolean("is_cta", &image->is_cta);
+  return true;
+}
+
+void DoodleFetcher::ParseBaseInformation(const base::DictionaryValue& ddljson,
+                                         DoodleConfig* config) const {
+  config->search_url = ParseRelativeUrl(ddljson, "search_url");
+  config->target_url = ParseRelativeUrl(ddljson, "target_url");
+  config->fullpage_interactive_url =
+      ParseRelativeUrl(ddljson, "fullpage_interactive_url");
+
+  config->doodle_type = ParseDoodleType(ddljson);
+  ddljson.GetString("alt_text", &config->alt_text);
+
+  // The JSON doesn't guarantee the number to fit into an int.
+  double ttl = 0;
+  if (!ddljson.GetDouble("time_to_live_ms", &ttl)) {
+    LOG(ERROR) << "No valid Doodle image TTL present in ddljson!";
+    return;
+  }

[Marc Treib, 2017/02/03 15:45:48]

The old search_provider_logos code also clamps the TTL to some reasonable max
range, so that if we once get a bad value (very far in the future), that doodle
won't be shown forever and ever. Maybe we should do something similar? (Not sure
if it's necessary)

[fhorschig, 2017/02/06 11:48:22]

It's easy to do but hard to define "too long".
The 125th birthday of doodle would be there too long if it stays
for a week.
The olympic fruits doodle would be rightfully there for three weeks.

There are checks and tests verifying that the doodle doesn't exceeds
30 days or expires back in time.

[Marc Treib, 2017/02/06 17:26:35]

I think just replicating the 30 day limit is fine (it seems to have worked well
in practice so far).

[jshneier, 2017/02/06 19:02:40]

1 day should be sufficient in just about all cases.  What's the behavior wrt the
ttl?  Does it still fetch a new config every time, or does it cache it for this
amount of time?  Is the ttl only for cases where you go offline?

[Marc Treib, 2017/02/07 09:21:09]

IIUC, the current behavior is: On each request from the UI, hand out the old
config unless it's expired, but then immediately fetch an update. So I think the
actual TTL limit doesn't matter too much.

[fhorschig, 2017/02/08 09:10:23]

Changed to 1 day.

[Marc Treib, 2017/02/08 10:51:36]

TBH, I'd prefer to keep the previous numbers. There's a small chance this change
will break something unexpectedly. Let's try to keep the differences between the
old and new systems limited to only the things we explicitly want to change.

[fhorschig, 2017/02/08 18:37:25]

Done.

+  config->expiry_date = clock_->Now() + base::TimeDelta::FromMillisecondsD(ttl);
+}
+
+GURL DoodleFetcher::ParseRelativeUrl(const base::DictionaryValue& dict_value,
+                                     const std::string& key) const {
+  std::string str_url;
+  dict_value.GetString(key, &str_url);
+  if (str_url.empty()) {
+    return GURL();
+  }
+  return GetGoogleBaseUrl().Resolve(str_url);
+}
+
+void DoodleFetcher::RespondToAllCallbacks(DoodleState state,
+                                          base::Optional<DoodleConfig> config) {
+  for (auto& callback : callbacks_) {
+    std::move(callback).Run(state, std::move(config));

[Marc Treib, 2017/02/03 15:45:48]

This will break if there's more than one callback: The second one won't get a
config anymore, since it's already been moved out.
Add a test for this? :)

[fhorschig, 2017/02/06 11:48:22]

It's const& now. There is no need to move that at all.

+  }
+  callbacks_.clear();
+}
+
+GURL DoodleFetcher::GetGoogleBaseUrl() const {
+  // There might be no tracker without profile. The factory returns nullptr in
+  // tests, too.
+  // TODO(fhorschig): DCHECK(google_url_tracker_) and inject a fake in tests.

[Marc Treib, 2017/02/03 15:45:48]

The nice thing is that we won't need the kDefaultGoogleHomepage anymore

[fhorschig, 2017/02/06 11:48:22]

It was already gone, resp. replaced by the public default of
the GoogleBaseUrlTracker.

+  if (google_url_tracker_) {
+    return google_url_tracker_->google_url();
+  }
+  GURL cmd_line_url = google_util::CommandLineGoogleBaseURL();

[Marc Treib, 2017/02/03 15:45:48]

This needs to go before the tracker.

[fhorschig, 2017/02/06 11:48:22]

Done.

+  if (cmd_line_url.is_valid()) {
+    return cmd_line_url;
+  }
+  return GURL(GoogleURLTracker::kDefaultGoogleHomepage);
+}
+
+}  // namespace doodle