Adding URLResponse Disk Cache to mojo.

services/service_cache/service_cache_impl.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "services/service_cache/service_cache_impl.h"
+
+#include "base/bind.h"
+#include "base/files/file_util.h"
+#include "base/location.h"
+#include "base/logging.h"
+#include "base/strings/string_util.h"
+#include "base/strings/stringprintf.h"
+#include "mojo/common/data_pipe_utils.h"
+#include "mojo/public/cpp/application/application_connection.h"
+#include "mojo/public/cpp/bindings/lib/fixed_buffer.h"
+#include "services/service_cache/service_cache_entry.mojom.h"
+#include "third_party/zlib/google/zip_reader.h"
+#include "url/gurl.h"
+
+namespace mojo {
+namespace service_cache {
+
+namespace {
+
+const char kEtagHeader[] = "etag";
+
+template <typename T>
+void Serialize(T input, std::string* output) {
+  typedef typename mojo::internal::WrapperTraits<T>::DataType DataType;
+  size_t size = GetSerializedSize_(input);
+  mojo::internal::FixedBuffer buf(size);
+  DataType data_type;
+  Serialize_(input.Pass(), &buf, &data_type);

[blundell, 2015/05/07 11:29:21]

The usage of these internal SDK functions is a little suspicious. Is there a
particular reason that the cache entry has to be a mojom struct? For example,
what about using //base/pickle?

[qsr, 2015/05/07 12:49:02]

 Versioning might be useful.

 Also, we have plan to make serializing/deserializing mojo struct more a first
class notion. I would move this code to use whatever comes from there when it
exists.

+  std::vector<Handle> handles;
+  data_type->EncodePointersAndHandles(&handles);
+  void* serialized_data = buf.Leak();
+  *output = std::string(static_cast<char*>(serialized_data), size);
+  free(serialized_data);
+}
+
+template <typename T>
+void Deserialize(std::string input, T* output) {
+  typedef typename mojo::internal::WrapperTraits<T>::DataType DataType;
+  DataType data_type = reinterpret_cast<DataType>(&input[0]);
+  std::vector<Handle> handles;
+  data_type->DecodePointersAndHandles(&handles);
+  Deserialize_(data_type, output);
+}
+
+Array<uint8_t> PathToArray(const base::FilePath& path) {
+  if (path.empty())
+    return Array<uint8_t>();
+  const std::string& string = path.value();
+  Array<uint8_t> result(string.size());
+  memcpy(&result.front(), string.data(), string.size());
+  return result.Pass();
+}
+
+std::string EncodeString(const std::string& string) {
+  std::string result = "";
+  for (size_t i = 0; i < string.size(); ++i) {
+    unsigned char c = string[i];
+    if (c >= 32 && c < 128 && c != '#' && c != ':') {
+      result += c;
+    } else {
+      result += base::StringPrintf("#%02x", c);
+    }
+  }
+  return result;
+}
+
+base::FilePath GetBaseDirectory() {
+  return base::FilePath(getenv("HOME")).Append(".mojo_service_cache");
+}
+
+base::FilePath GetDirName(base::FilePath base_directory,
+                          const std::string& url) {
+  // TODO(qsr): If the speed of directory traversal is problematic, this might
+  // need to change to use less directories.
+  return base_directory.Append(EncodeString(url));
+}
+
+base::FilePath GetConsumerCacheDirectory(const base::FilePath& main_cache) {
+  return main_cache.Append("consumer_cache");
+}
+
+base::FilePath GetExtractedSentinel(const base::FilePath& main_cache) {
+  return main_cache.Append("extracted_sentinel");
+}
+
+void RunCallbackWithSuccess(
+    const ServiceCacheImpl::FilePathPairCallback& callback,
+    const base::FilePath& content_path,
+    const base::FilePath& cache_dir,
+    const base::FilePath& entry_path,
+    CacheEntryPtr entry,
+    bool success) {
+  if (!success) {
+    callback.Run(base::FilePath(), base::FilePath());
+    return;
+  }
+  std::string serialize_entry;

[blundell, 2015/05/07 11:29:21]

nit: s/serialize_entry/serialized_entry

[qsr, 2015/05/07 12:49:02]

Done.

+  Serialize(entry.Pass(), &serialize_entry);
+  // We can ignore error, as it will just force to clear the cache on the next
+  // request.
+  WriteFile(entry_path, serialize_entry.data(), serialize_entry.size());
+  callback.Run(content_path, cache_dir);
+}
+
+void RunMojoCallback(
+    const Callback<void(Array<uint8_t>, Array<uint8_t>)>& callback,
+    const base::FilePath& path1,
+    const base::FilePath& path2) {
+  callback.Run(PathToArray(path1), PathToArray(path2));
+}
+
+std::vector<std::string> GetHeaderValues(const std::string& header_name,
+                                         const Array<String>& headers) {
+  std::vector<std::string> result;
+  for (std::string header : headers.storage()) {
+    if (StartsWithASCII(header, header_name, false)) {
+      auto begin = header.begin();
+      auto end = header.end();
+      begin += header_name.size();
+      while (begin < end && *begin == ' ')

[blundell, 2015/05/07 11:29:21]

Can you use //base/string_util's TrimString() function here to make this
cleaner?

[qsr, 2015/05/07 12:49:02]

 I could. But this would be a lot less efficient, as I would need to create a
bunch of intermediary strings with copy all over the place...

 To be noted -> jam is planning to change this API so that headers will be a
struct with a name and a value. When this happens, this code will change for the
better.

+        begin++;
+      if (begin < end && *begin == ':') {
+        begin++;
+        while (begin < end && *begin == ' ')
+          begin++;
+        while (end > begin && *(end - 1) == ' ')
+          end--;
+        if (begin < end) {
+          result.push_back(std::string(begin, end));
+        }
+      }
+    }
+  }
+  return result;
+}
+
+bool IsCacheEntryValid(const base::FilePath& dir,
+                       URLResponse* response,
+                       CacheEntryPtr* output) {
+  base::FilePath entry_path = dir.Append("entry");
+  if (!base::PathExists(entry_path))
+    return false;
+  std::string serialized_entry;
+  if (!ReadFileToString(entry_path, &serialized_entry))
+    return false;
+  CacheEntryPtr entry;
+  Deserialize(serialized_entry, &entry);
+  if (entry->headers.is_null() || response->headers.is_null())
+    return false;
+
+  // Only handle etag for the moment.
+  std::string etag_header_name = kEtagHeader;
+  std::vector<std::string> entry_etags =
+      GetHeaderValues(etag_header_name, entry->headers);
+  if (entry_etags.size() == 0)
+    return false;
+  std::vector<std::string> response_etags =
+      GetHeaderValues(etag_header_name, response->headers);
+  if (response_etags.size() == 0)
+    return false;
+
+  // Looking for the first etag header.

[blundell, 2015/05/07 11:29:21]

Will there ever be more than 1 etag header? What's the semantics in that case?

[qsr, 2015/05/07 12:49:02]

 There is no defined semantic when there is more than one etag header. Chrome is
also looking for the first one.

+  bool result = entry_etags[0] == response_etags[0];
+
+  if (output)
+    *output = entry.Pass();
+  return result;
+}
+
+}  // namespace
+
+ServiceCacheImpl::ServiceCacheImpl(
+    scoped_refptr<base::SequencedWorkerPool> worker_pool,
+    const std::string& remote_application_url,
+    InterfaceRequest<ServiceCache> request)
+    : worker_pool_(worker_pool), binding_(this, request.Pass()) {
+  base_directory_ = GetBaseDirectory();
+  if (remote_application_url != "") {
+    base_directory_ = base_directory_.Append(
+        EncodeString(GURL(remote_application_url).GetOrigin().spec()));
+  }
+}
+
+ServiceCacheImpl::~ServiceCacheImpl() {
+}
+
+void ServiceCacheImpl::GetFile(URLResponsePtr response,
+                               const GetFileCallback& callback) {
+  return GetFileInternal(response.Pass(),
+                         base::Bind(&RunMojoCallback, callback));
+}
+
+void ServiceCacheImpl::GetExtractedContent(
+    URLResponsePtr response,
+    const GetExtractedContentCallback& callback) {
+  base::FilePath dir = GetDirName(base_directory_, response->url);
+  if (dir.empty()) {

[blundell, 2015/05/07 11:29:21]

Why won't this be the case the first time that this method is called for
|response|?

[blundell, 2015/05/07 11:33:20]

OK, in the case of a content-handled application it won't since the shell
already fetched the initial URLResponse (i.e., with the shbang prepended) via
GetFile(). That doesn't seem like it generalizes though?

[qsr, 2015/05/07 12:49:02]

Removed this test as this cannot happen. empty() has not the semantic you think
-> it just check that the path itself is not empty, not that the path exists on
disk.

[qsr, 2015/05/07 12:49:02]

Acknowledged.

+    callback.Run(Array<uint8_t>(), Array<uint8_t>());
+    return;
+  }
+  base::FilePath extracted_dir = dir.Append("extracted");
+  if (IsCacheEntryValid(dir, response.get(), nullptr) &&
+      PathExists(GetExtractedSentinel(dir))) {
+    callback.Run(PathToArray(extracted_dir), PathToArray(dir));
+    return;
+  }
+
+  GetFileInternal(
+      response.Pass(),
+      base::Bind(&ServiceCacheImpl::GetExtractedContentInternal,
+                 base::Unretained(this), base::Bind(&RunMojoCallback, callback),
+                 extracted_dir));
+}
+
+void ServiceCacheImpl::GetFileInternal(URLResponsePtr response,
+                                       const FilePathPairCallback& callback) {
+  base::FilePath dir = GetDirName(base_directory_, response->url);
+  CacheEntryPtr entry;
+  if (IsCacheEntryValid(dir, response.get(), &entry)) {
+    callback.Run(base::FilePath(entry->content_path),
+                 GetConsumerCacheDirectory(dir));
+    return;
+  }
+  if (base::PathExists(dir)) {
+    // Clear cached data.
+    base::FilePath to_delete;
+    CHECK(CreateTemporaryDirInDir(base_directory_, "to_delete", &to_delete));
+    CHECK(Move(dir, to_delete));
+    worker_pool_->PostTask(
+        FROM_HERE,
+        base::Bind(base::IgnoreResult(&base::DeleteFile), to_delete, true));
+  }
+  if (!response->body.is_valid() ||
+      !base::CreateDirectoryAndGetError(dir, nullptr) ||
+      !base::CreateDirectoryAndGetError(GetConsumerCacheDirectory(dir),
+                                        nullptr)) {
+    callback.Run(base::FilePath(), base::FilePath());
+    return;
+  }
+  base::FilePath entry_path = dir.Append("entry");
+  base::FilePath content;
+  CHECK(CreateTemporaryFileInDir(dir, &content));
+  entry = CacheEntry::New();
+  entry->url = response->url;
+  entry->content_path = content.value();
+  entry->headers = response->headers.Pass();
+  common::CopyToFile(response->body.Pass(), content, worker_pool_.get(),
+                     base::Bind(&RunCallbackWithSuccess, callback, content,
+                                GetConsumerCacheDirectory(dir), entry_path,
+                                base::Passed(entry.Pass())));
+}
+
+void ServiceCacheImpl::GetExtractedContentInternal(
+    const FilePathPairCallback& callback,
+    const base::FilePath& extracted_dir,
+    const base::FilePath& content,
+    const base::FilePath& dir) {
+  if (content.empty()) {
+    callback.Run(base::FilePath(), base::FilePath());
+    return;
+  }
+
+  zip::ZipReader reader;
+  if (!reader.Open(content)) {
+    callback.Run(base::FilePath(), base::FilePath());
+    return;
+  }
+  while (reader.HasMore()) {
+    bool success = reader.OpenCurrentEntryInZip();
+    success = success && reader.ExtractCurrentEntryIntoDirectory(extracted_dir);
+    success = success && reader.AdvanceToNextEntry();
+    if (!success) {
+      callback.Run(base::FilePath(), base::FilePath());
+      return;
+    }
+  }
+  // We can ignore error, as it will just force to clear the cache on the next
+  // request.
+  WriteFile(GetExtractedSentinel(dir), nullptr, 0);
+  callback.Run(extracted_dir, GetConsumerCacheDirectory(dir));
+}
+
+}  // namespace service_cache
+}  // namespace mojo