Fetch snippets from ChromeReader and show them on the NTP

components/ntp_snippets/ntp_snippets_fetcher.cc

Index: components/ntp_snippets/ntp_snippets_fetcher.cc
diff --git a/components/ntp_snippets/ntp_snippets_fetcher.cc b/components/ntp_snippets/ntp_snippets_fetcher.cc
new file mode 100644
index 0000000000000000000000000000000000000000..9ec67aa1de4fea154cf29624d0fca41b52717e5e
--- /dev/null
+++ b/components/ntp_snippets/ntp_snippets_fetcher.cc
@@ -0,0 +1,190 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+#include "components/ntp_snippets/ntp_snippets_fetcher.h"

[mmenke, 2016/02/18 16:51:11]

nit:  blank line before first include.

[May, 2016/02/19 14:49:02]

Done.

+
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "base/strings/stringprintf.h"
+#include "base/task_runner_util.h"
+#include "components/signin/core/browser/profile_oauth2_token_service.h"
+#include "components/signin/core/browser/signin_manager.h"
+#include "components/signin/core/browser/signin_tracker.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_request_headers.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_status_code.h"
+#include "net/url_request/url_fetcher.h"
+
+using net::URLFetcher;
+using net::URLRequestContextGetter;
+using net::HttpRequestHeaders;
+using net::URLRequestStatus;
+
+namespace ntp_snippets {
+
+const char kSnippetSuggestionsFilename[] = "ntp_snippets.json";
+const char kApiScope[] = "https://www.googleapis.com/auth/webhistory";
+const char kContentSnippetsServer[] =
+    "https://chromereader-pa.googleapis.com/v1/fetch";
+const char kAuthorizationRequestHeaderFormat[] = "Bearer %s";
+
+const char kUnpersonalizedRequestParameters[] =
+    "{ \"response_detail_level\": \"FULL_DEBUG\", \"advanced_options\": { "
+    "\"local_scoring_params\": {\"content_params\" : { "
+    "\"only_return_personalized_results\": false } }, "
+    "\"global_scoring_params\": { \"num_to_return\": 10 } } }";
+
+base::FilePath GetSnippetsSuggestionsPath(const base::FilePath& base_dir) {
+  return base_dir.AppendASCII(kSnippetSuggestionsFilename);
+}
+
+NTPSnippetsFetcher::NTPSnippetsFetcher(
+    scoped_refptr<base::SequencedTaskRunner> file_task_runner,
+    SigninManager* signin_manager,
+    OAuth2TokenService* token_service,
+    URLRequestContextGetter* url_request_context_getter,
+    const base::FilePath& base_download_path)
+    : OAuth2TokenService::Consumer("NTP_snippets"),
+      file_task_runner_(file_task_runner),
+      url_request_context_getter_(url_request_context_getter),
+      signin_manager_(signin_manager),
+      token_service_(token_service),
+      download_path_(GetSnippetsSuggestionsPath(base_download_path)),
+      waiting_for_refresh_token_(false),
+      weak_ptr_factory_(this) {}
+
+NTPSnippetsFetcher::~NTPSnippetsFetcher() {
+  if (waiting_for_refresh_token_)
+    token_service_->RemoveObserver(this);
+}
+
+scoped_ptr<NTPSnippetsFetcher::SnippetsAvailableCallbackList::Subscription>
+NTPSnippetsFetcher::AddCallback(const SnippetsAvailableCallback& callback) {
+  return callback_list_.Add(callback);
+}
+
+void NTPSnippetsFetcher::FetchSnippets(bool overwrite) {
+  if (overwrite) {
+    StartFetch();
+  } else {
+    base::PostTaskAndReplyWithResult(
+        file_task_runner_.get(), FROM_HERE,
+        base::Bind(&base::PathExists, download_path_),
+        base::Bind(&NTPSnippetsFetcher::OnFileExistsCheckDone,
+                   weak_ptr_factory_.GetWeakPtr()));
+  }
+}
+
+void NTPSnippetsFetcher::OnFileExistsCheckDone(bool exists) {
+  if (exists) {
+    NotifyObservers();
+  } else {
+    StartFetch();
+  }
+}
+
+void NTPSnippetsFetcher::StartFetch() {
+  if (signin_manager_->IsAuthenticated()) {
+    StartTokenRequest();
+  } else {
+    if (!waiting_for_refresh_token_) {
+      // Wait until we get a refresh token.
+      waiting_for_refresh_token_ = true;
+      token_service_->AddObserver(this);
+    }
+  }
+}
+
+void NTPSnippetsFetcher::StartTokenRequest() {
+  OAuth2TokenService::ScopeSet scopes;
+  scopes.insert(kApiScope);
+  oauth_request_ = token_service_->StartRequest(
+      signin_manager_->GetAuthenticatedAccountId(), scopes, this);
+}
+
+void NTPSnippetsFetcher::NotifyObservers() {
+  callback_list_.Notify(download_path_);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// OAuth2TokenService::Consumer overrides
+void NTPSnippetsFetcher::OnGetTokenSuccess(
+    const OAuth2TokenService::Request* request,
+    const std::string& access_token,
+    const base::Time& expiration_time) {
+  oauth_request_.reset();
+  url_fetcher_ =
+      URLFetcher::Create(GURL(kContentSnippetsServer), URLFetcher::POST, this);
+  url_fetcher_->SetRequestContext(url_request_context_getter_);
+  url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
+                             net::LOAD_DO_NOT_SAVE_COOKIES);
+  HttpRequestHeaders headers;
+  headers.SetHeader("Authorization",
+                    base::StringPrintf(kAuthorizationRequestHeaderFormat,
+                                       access_token.c_str()));
+  headers.SetHeader("Content-Type", "application/json; charset=UTF-8");
+  url_fetcher_->SetExtraRequestHeaders(headers.ToString());
+  url_fetcher_->SetUploadData("application/json",
+                              kUnpersonalizedRequestParameters);
+  url_fetcher_->SaveResponseToTemporaryFile(file_task_runner_.get());
+  url_fetcher_->Start();

[mmenke, 2016/02/18 16:51:11]

Note that url_fetcher_ has no timeout logic (Other than TCP/HTTP2/QUIC
keepalives, if enabled on the platform), so this could just hand indefinitely,
like when behind a captive portal that blackholes SSL requests.

It also has no size cap, so could happily download a 15 terabyte file, though
taking to a trusted server over SSL helps alleviate that potential issue.

[May, 2016/02/19 14:49:02]

Acknowledged. Is there a different way to implement this, to address those?
(Though like you said -- we would only ever fetch from a trusted Google server
using this, so the second one is not as big of a concern).

[mmenke, 2016/02/19 15:42:22]

The only way to address the size issue currently is to use the URLRequest
interface (Which is a fair bit more complicated to use, and requires you use it
on the IO thread).  I'd like to get a hard limit in URLFetcher, but given the
sheer number of consumers, that's a pretty major task.  We could go with a way
to set an optional limit until then, but I'd really like to switch to have a
sane default limit by default (and it should not be possible to increase it, at
least not by much).

For the timeout issue, there's no current workaround, other than to do it
yourself.  The network stack doesn't have any context, so it doesn't know what
is reasonable.  It could be a hanging get, for instance.  We do handle timeouts
while establishing a connection, and use QUIC/HTTP2/TCP keepalives, if enabled
(TCP keepalives are turned on for desktop, disabled for mobile.  Think the
others are always on), but nothing beyond that.

+}
+
+void NTPSnippetsFetcher::OnGetTokenFailure(
+    const OAuth2TokenService::Request* request,
+    const GoogleServiceAuthError& error) {
+  oauth_request_.reset();
+  DLOG(ERROR) << "Unable to get token: " << error.ToString();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// OAuth2TokenService::Observer overrides
+void NTPSnippetsFetcher::OnRefreshTokenAvailable(
+    const std::string& account_id) {
+  token_service_->RemoveObserver(this);
+  waiting_for_refresh_token_ = false;
+  StartTokenRequest();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// URLFetcherDelegate overrides
+void NTPSnippetsFetcher::OnURLFetchComplete(const URLFetcher* source) {
+  DCHECK_EQ(url_fetcher_.get(), source);
+
+  const URLRequestStatus& status = source->GetStatus();
+  if (!status.is_success()) {
+    DLOG(WARNING) << "URLRequestStatus error " << status.error()
+                  << " while trying to download " << source->GetURL().spec();
+    return;
+  }
+
+  int response_code = source->GetResponseCode();
+  if (response_code != net::HTTP_OK) {
+    DLOG(WARNING) << "HTTP error " << response_code
+                  << " while trying to download " << source->GetURL().spec();
+    return;
+  }
+
+  base::FilePath response_path;
+  source->GetResponseAsFilePath(false, &response_path);
+
+  base::PostTaskAndReplyWithResult(
+      file_task_runner_.get(), FROM_HERE,
+      base::Bind(&base::Move, response_path, download_path_),
+      base::Bind(&NTPSnippetsFetcher::OnFileMoveDone,
+                 weak_ptr_factory_.GetWeakPtr()));
+}
+
+void NTPSnippetsFetcher::OnFileMoveDone(bool success) {
+  if (!success) {
+    DLOG(WARNING) << "Could not move file to "
+                  << download_path_.LossyDisplayName();
+    return;
+  }
+
+  NotifyObservers();
+}
+
+}  // namespace ntp_snippets