Certificate Transparency: Add STH Fetching capability.

components/certificate_transparency/log_proof_fetcher_unittest.cc

Index: components/certificate_transparency/log_proof_fetcher_unittest.cc
diff --git a/components/certificate_transparency/log_proof_fetcher_unittest.cc b/components/certificate_transparency/log_proof_fetcher_unittest.cc
new file mode 100644
index 0000000000000000000000000000000000000000..df1459cdb6013037b767ceb1dc2d5329820b2aa3
--- /dev/null
+++ b/components/certificate_transparency/log_proof_fetcher_unittest.cc
@@ -0,0 +1,299 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/certificate_transparency/log_proof_fetcher.h"
+
+#include <string>
+
+#include "components/safe_json/testing_json_parser.h"
+#include "net/base/net_errors.h"
+#include "net/base/network_delegate.h"
+#include "net/cert/signed_tree_head.h"
+#include "net/http/http_status_code.h"
+#include "net/test/ct_test_util.h"
+#include "net/url_request/url_request_context.h"
+#include "net/url_request/url_request_filter.h"
+#include "net/url_request/url_request_interceptor.h"
+#include "net/url_request/url_request_job.h"
+#include "net/url_request/url_request_test_job.h"
+#include "net/url_request/url_request_test_util.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace certificate_transparency {
+
+namespace {
+
+const char kGetSTHHeaders[] =
+    "HTTP/1.1 200 OK\0"
+    "Content-Type: application/json; charset=ISO-8859-1\0"
+    "\0";

[mmenke, 2015/08/05 15:37:09]

nit:  Second \0 at the end not needed (C strings are always NULL terminated, and
you just need two at the end, not 3)

[Eran Messeri, 2015/08/06 09:07:51]

Done.

+
+const char kGetSTHNotFoundHeaders[] =
+    "HTTP/1.1 404 Not Found\0"
+    "Content-Type: text/html; charset=iso-8859-1\0"
+    "\0";

[mmenke, 2015/08/05 15:37:09]

nit:  Remove last \0

[Eran Messeri, 2015/08/06 09:07:51]

Done.

+
+const char kLogSchema[] = "https";
+const char kLogURL[] = "ct.log.example.com";
+const char kLogID[] = "some_id";
+
+class FetchSTHTestJob : public net::URLRequestTestJob {
+ public:
+  FetchSTHTestJob(const std::string& get_sth_data,
+                  const std::string& get_sth_headers,
+                  net::URLRequest* request,
+                  net::NetworkDelegate* network_delegate)
+      : URLRequestTestJob(request,
+                          network_delegate,
+                          get_sth_headers,
+                          get_sth_data,
+                          true),
+        async_io_(false) {}
+
+  void set_async_io(bool async_io) { async_io_ = async_io; }
+
+ private:
+  bool NextReadAsync() override {
+    // Response with indication of async IO only once, otherwise the final
+    // Read would (incorrectly) be classified as async, causing the
+    // URLRequestJob to try reading another time and failing on a CHECK
+    // that the raw_read_buffer_ is not null.
+    if (async_io_) {
+      async_io_ = false;
+      return true;
+    }
+    return false;
+  }
+
+  ~FetchSTHTestJob() override {}

[mmenke, 2015/08/05 15:37:09]

nit:  Destructors should go before all other methods in their section (Except
constructors, of couse)

[Eran Messeri, 2015/08/06 09:07:51]

Done.

+
+  bool async_io_;
+
+  DISALLOW_COPY_AND_ASSIGN(FetchSTHTestJob);
+};
+
+class GetSTHResponseHandler : public net::URLRequestInterceptor {
+ public:
+  GetSTHResponseHandler()
+      : async_io_(false),
+        response_body_(""),
+        response_headers_(
+            std::string(kGetSTHHeaders, arraysize(kGetSTHHeaders))) {}
+  ~GetSTHResponseHandler() override {}
+
+  // URLRequestInterceptor implementation:
+  net::URLRequestJob* MaybeInterceptRequest(
+      net::URLRequest* request,
+      net::NetworkDelegate* network_delegate) const override {
+    DVLOG(1) << "MaybeInterceptRequest, headers: " << response_headers_;

[mmenke, 2015/08/05 15:37:09]

Should we check that request->url() is the URL we expect?  Or instead of using
"AddHostnameInterceptor", use add AddUrlInterceptor, so we only catch the one
URL we should be seeing (Nice thing about an EXPECT_EQ line here, though, is
displays exactly what the incorrect URL was that we requested, if things are
broken).

Also maybe throw in some fun stuff in the original URL, if it doesn't break the
expected_sth checks?  "https://spam:10042/whatever/", and check that the
expected path is requested?

[Eran Messeri, 2015/08/06 09:07:51]

Done - added an EXPECT_EQ check for the URL.
Note that the code under test constructs the full URL (i.e. appending
/ct/v1/get-sth) according to RFC6962, so checking the received URL is quite
useful, but there's limited space for playing with the original URL.
I did add another component to the log's path to emulate a real log's address.

+    FetchSTHTestJob* job = new FetchSTHTestJob(
+        response_body_, response_headers_, request, network_delegate);
+    job->set_async_io(async_io_);
+    return job;
+  }
+
+  void set_response_body(std::string response_body) {
+    response_body_ = response_body;
+  }
+
+  void set_response_headers(std::string response_headers) {
+    response_headers_ = response_headers;
+  }
+
+  void set_async_io(bool async_io) { async_io_ = async_io; }
+
+ private:
+  bool async_io_;
+  std::string response_body_;
+  std::string response_headers_;
+
+  DISALLOW_COPY_AND_ASSIGN(GetSTHResponseHandler);
+};
+
+class RecordFetchCallbackInvocations {
+ public:
+  RecordFetchCallbackInvocations(bool expect_success)
+      : expect_success_(expect_success),
+        invoked_(false),
+        net_error_(net::OK),
+        http_response_code_(-1) {}
+
+  void STHFetched(const std::string& log_id,
+                  const net::ct::SignedTreeHead& sth) {
+    ASSERT_TRUE(expect_success_);
+    ASSERT_FALSE(invoked_);
+    invoked_ = true;
+    // If expected to succeed, expecting the known_good STH.
+    net::ct::SignedTreeHead expected_sth;
+    net::ct::GetSampleSignedTreeHead(&expected_sth);
+
+    ASSERT_EQ(kLogID, log_id);
+    ASSERT_EQ(expected_sth.version, sth.version);
+    ASSERT_EQ(expected_sth.timestamp, sth.timestamp);
+    ASSERT_EQ(expected_sth.tree_size, sth.tree_size);
+    ASSERT_STREQ(expected_sth.sha256_root_hash, sth.sha256_root_hash);
+    ASSERT_EQ(expected_sth.signature.hash_algorithm,
+              sth.signature.hash_algorithm);
+    ASSERT_EQ(expected_sth.signature.signature_algorithm,
+              sth.signature.signature_algorithm);
+    ASSERT_EQ(expected_sth.signature.signature_data,
+              sth.signature.signature_data);
+  }
+
+  void FetchingFailed(const std::string& log_id,
+                      int net_error,
+                      int http_response_code) {
+    ASSERT_FALSE(expect_success_);
+    ASSERT_FALSE(invoked_);
+    invoked_ = true;
+    net_error_ = net_error;
+    http_response_code_ = http_response_code;
+    if (net_error_ == net::OK) {
+      ASSERT_NE(net::HTTP_OK, http_response_code_);
+    }
+  }
+
+  bool invoked() const { return invoked_; }
+
+  int net_error() const { return net_error_; }
+
+  int http_response_code() const { return http_response_code_; }
+
+ private:
+  const bool expect_success_;
+  bool invoked_;
+  int net_error_;
+  int http_response_code_;
+};
+
+class LogProofFetcherTest : public ::testing::Test {
+ public:
+  LogProofFetcherTest()
+      : log_url_(std::string(kLogSchema) + "://" + std::string(kLogURL) + "/") {
+    scoped_ptr<GetSTHResponseHandler> handler(new GetSTHResponseHandler());
+    handler_ = handler.get();
+
+    net::URLRequestFilter::GetInstance()->AddHostnameInterceptor(
+        kLogSchema, kLogURL, handler.Pass());
+
+    fetcher_.reset(new LogProofFetcher(&context_));
+  }
+
+  ~LogProofFetcherTest() override {
+    net::URLRequestFilter::GetInstance()->RemoveHostnameHandler(kLogSchema,
+                                                                kLogURL);
+  }
+
+ protected:
+  void SetValidSTHJSONResponse() {
+    std::string sth_json_reply_data = net::ct::GetSampleSTHAsJson();
+    handler_->set_response_body(sth_json_reply_data);
+  }
+
+  void RunFetcherWithCallback(RecordFetchCallbackInvocations* callback) {
+    fetcher_->FetchSignedTreeHead(
+        log_url_, kLogID,
+        base::Bind(&RecordFetchCallbackInvocations::STHFetched,
+                   base::Unretained(callback)),
+        base::Bind(&RecordFetchCallbackInvocations::FetchingFailed,
+                   base::Unretained(callback)));
+    message_loop_.RunUntilIdle();
+  }
+
+  base::MessageLoopForIO message_loop_;
+  net::TestURLRequestContext context_;
+  safe_json::TestingJsonParser::ScopedFactoryOverride factory_override_;
+  scoped_ptr<LogProofFetcher> fetcher_;
+  const GURL log_url_;
+  GetSTHResponseHandler* handler_;
+};
+
+TEST_F(LogProofFetcherTest, TestValidGetReply) {
+  SetValidSTHJSONResponse();
+
+  RecordFetchCallbackInvocations callback(true);
+
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());

[mmenke, 2015/08/05 15:37:09]

optional:  Think these can all be EXPECTs instead, particularly the ones other
than callback.invoked().

[Eran Messeri, 2015/08/06 09:07:51]

Done.

+}
+
+TEST_F(LogProofFetcherTest, TestValidGetReplyAsyncIO) {
+  SetValidSTHJSONResponse();
+  handler_->set_async_io(true);
+
+  RecordFetchCallbackInvocations callback(true);
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());
+}
+
+TEST_F(LogProofFetcherTest, TestInvalidGetReplyIncompleteJSON) {
+  std::string sth_json_reply_data = net::ct::CreateSignedTreeHeadJsonString(
+      21 /* tree_size */, 123456u /* timestamp */, std::string(),
+      std::string());
+  handler_->set_response_body(sth_json_reply_data);
+
+  RecordFetchCallbackInvocations callback(false);
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());
+  ASSERT_EQ(net::ERR_CT_STH_INCOMPLETE, callback.net_error());
+}
+
+TEST_F(LogProofFetcherTest, TestInvalidGetReplyInvalidJSON) {
+  std::string sth_json_reply_data = "{\"tree_size\":21,\"timestamp\":}";
+  handler_->set_response_body(sth_json_reply_data);
+
+  RecordFetchCallbackInvocations callback(false);
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());
+  ASSERT_EQ(net::ERR_CT_STH_PARSING_FAILED, callback.net_error());
+}
+
+TEST_F(LogProofFetcherTest, TestLogReplyIsTooLong) {
+  std::string sth_json_reply_data = net::ct::GetSampleSTHAsJson();
+  // Add kMaxLogResponseSizeInBytes to make sure the response is too big.
+  sth_json_reply_data.append(
+      std::string(LogProofFetcher::kMaxLogResponseSizeInBytes, ' '));
+  handler_->set_response_body(sth_json_reply_data);
+
+  RecordFetchCallbackInvocations callback(false);
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());
+  ASSERT_EQ(net::ERR_FILE_TOO_BIG, callback.net_error());
+  ASSERT_EQ(net::HTTP_OK, callback.http_response_code());
+}
+
+TEST_F(LogProofFetcherTest, TestLogReplyIsExactlyMaxSize) {
+  std::string sth_json_reply_data = net::ct::GetSampleSTHAsJson();
+  // Extend the reply to be exactly kMaxLogResponseSizeInBytes.
+  sth_json_reply_data.append(std::string(
+      LogProofFetcher::kMaxLogResponseSizeInBytes - sth_json_reply_data.size(),
+      ' '));
+  handler_->set_response_body(sth_json_reply_data);
+
+  RecordFetchCallbackInvocations callback(true);
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());
+}
+
+TEST_F(LogProofFetcherTest, TestLogRepliesWithHttpError) {
+  handler_->set_response_headers(
+      std::string(kGetSTHNotFoundHeaders, arraysize(kGetSTHNotFoundHeaders)));
+
+  RecordFetchCallbackInvocations callback(false);
+  RunFetcherWithCallback(&callback);
+
+  ASSERT_TRUE(callback.invoked());
+  ASSERT_EQ(net::OK, callback.net_error());
+  ASSERT_EQ(net::HTTP_NOT_FOUND, callback.http_response_code());
+}
+
+}  // namespace
+
+}  // namespace certificate_transparency