[WIP][tracing] Add memory dump provider for sync Directory

sync/syncable/directory.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "sync/syncable/directory.h"
 
+#include <inttypes.h>
 #include <stddef.h>
 #include <stdint.h>
 
 #include <algorithm>
 #include <iterator>
 #include <utility>
 
 #include "base/base64.h"
 #include "base/guid.h"
 #include "base/metrics/histogram.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
+#include "base/strings/stringprintf.h"
+#include "base/trace_event/memory_dump_manager.h"
+#include "base/trace_event/process_memory_dump.h"
 #include "base/trace_event/trace_event.h"
 #include "sync/internal_api/public/base/attachment_id_proto.h"
 #include "sync/internal_api/public/base/unique_position.h"
 #include "sync/internal_api/public/util/unrecoverable_error_handler.h"
+#include "sync/protocol/proto_value_conversions.h"
 #include "sync/syncable/entry.h"
 #include "sync/syncable/entry_kernel.h"
 #include "sync/syncable/in_memory_directory_backing_store.h"
 #include "sync/syncable/model_neutral_mutable_entry.h"
 #include "sync/syncable/on_disk_directory_backing_store.h"
 #include "sync/syncable/scoped_kernel_lock.h"
 #include "sync/syncable/scoped_parent_child_index_updater.h"
 #include "sync/syncable/syncable-inl.h"
 #include "sync/syncable/syncable_base_transaction.h"
 #include "sync/syncable/syncable_changes_version.h"
 #include "sync/syncable/syncable_read_transaction.h"
 #include "sync/syncable/syncable_util.h"
 #include "sync/syncable/syncable_write_transaction.h"
 
 using std::string;
 
 namespace syncer {
 namespace syncable {
 
+namespace {
+
+template <typename Key, typename Value>
+size_t GetMapMemoryUsage(const std::unordered_map<Key, Value> map) {
+  return map.size() * (sizeof(Key) + sizeof(Value));

[stanisc, 2016/07/06 21:50:32]

This is too simplistic. See
http://info.prelert.com/blog/stl-container-memory-usage

[ssid, 2016/07/19 22:10:35]

Made it more accurate. Apart from these there are a few compiler specific usages
which are ignored here.

+}
+}
+
 // static
 const base::FilePath::CharType Directory::kSyncDatabaseFilename[] =
     FILE_PATH_LITERAL("SyncData.sqlite3");
 
 Directory::PersistedKernelInfo::PersistedKernelInfo() {
   ModelTypeSet protocol_types = ProtocolTypes();
   for (ModelTypeSet::Iterator iter = protocol_types.First(); iter.Good();
        iter.Inc()) {
     ResetDownloadProgress(iter.Get());
     transaction_version[iter.Get()] = 0;
@@ skipping 156 matching lines @@
   // Save changes back in case there are any metahandles to purge.
   if (!SaveChanges())
     return FAILED_INITIAL_WRITE;
 
   // Now that we've successfully opened the store, install an error handler to
   // deal with catastrophic errors that may occur later on. Use a weak pointer
   // because we cannot guarantee that this Directory will outlive the Closure.
   store_->SetCatastrophicErrorHandler(base::Bind(
       &Directory::OnCatastrophicError, weak_ptr_factory_.GetWeakPtr()));
 
+  UMA_HISTOGRAM_COUNTS("Sync.DirectoryMemoryUsageKB",

[stanisc, 2016/07/06 21:50:32]

Someone from the current sync team should comment on this. I think the original
idea was to get distribution by sync type but perhaps this should be OK.

[ssid, 2016/07/19 22:10:35]

Maybe that can be added later on top of this implementation. Should be easy to
add

+                       GetApproximateMemoryUsage());
+
   return OPENED;
 }
 
 DeleteJournal* Directory::delete_journal() {
   DCHECK(delete_journal_.get());
   return delete_journal_.get();
 }
 
 void Directory::Close() {
   store_.reset();
@@ skipping 685 matching lines @@
   ScopedKernelLock lock(this);
   kernel_->persisted_info.download_progress[model_type].SerializeToString(
       value_out);
 }
 
 size_t Directory::GetEntriesCount() const {
   ScopedKernelLock lock(this);
   return kernel_->metahandles_map.size();
 }
 
+size_t Directory::GetApproximateMemoryUsage() {
+  size_t total = 0;
+  ScopedKernelLock lock(this);
+  total += sizeof(this) + GetMapMemoryUsage(kernel_->metahandles_map) +
+           GetMapMemoryUsage(kernel_->ids_map) +
+           GetMapMemoryUsage(kernel_->server_tags_map) +
+           GetMapMemoryUsage(kernel_->client_tags_map) +
+           GetMapMemoryUsage(kernel_->index_by_attachment_id);
+  total += kernel_->parent_child_index.bytes_used();
+
+  for (size_t i = 0; i < MODEL_TYPE_COUNT; ++i)
+    total += kernel_->persisted_info.datatype_context[i].context().capacity();
+
+  for (auto entry : kernel_->index_by_attachment_id)
+    total += entry.first.capacity() + entry.second.size() * sizeof(int64_t);
+
+  for (auto handle : kernel_->metahandles_map) {
+    if (handle.second->cached_size()) {

[stanisc, 2016/07/06 21:50:32]

A comment here would be useful

[ssid, 2016/07/19 22:10:35]

Done.

+      total += handle.second->cached_size();
+      continue;
+    }
+
+    size_t entry_size = 0;
+    for (unsigned i = ID_FIELDS_BEGIN; i < ID_FIELDS_END; ++i) {
+      // Twice because the |ids_map| stores a copy of these strings.
+      entry_size +=
+          2 * handle.second->ref(static_cast<IdField>(i)).value().capacity();

[stanisc, 2016/07/06 21:50:31]

I am not sure this is right. client ID might be the same as the server ID and
then they wouldn't be entered into ids_map twice. Also most of the parent IDs
are empty e.g. deref to the same shared empty string. I am not sure if their
capacity would be zero or not - worth double checking.

[ssid, 2016/07/19 22:10:35]

counting the ids_map separately. The string capacities are usually 10.

+    }
+    for (unsigned i = STRING_FIELDS_BEGIN; i < STRING_FIELDS_END; ++i) {
+      if (i == UNIQUE_SERVER_TAG || i == UNIQUE_CLIENT_TAG) {
+        // These are stored again in tags_map.

[stanisc, 2016/07/06 21:50:32]

The same is here - if the server tag is the same as the client tag they wouldn't
appear twice in tags_map.

[ssid, 2016/07/19 22:10:35]

This one will appear twice if the id is same. There are two maps,
client_tags_map and server_tags_map. Updated the comment.

+        entry_size +=
+            2 * handle.second->ref(static_cast<StringField>(i)).capacity();
+      } else {
+        entry_size +=
+            handle.second->ref(static_cast<StringField>(i)).capacity();
+      }
+    }
+    for (unsigned i = PROTO_FIELDS_BEGIN; i < PROTO_FIELDS_END; ++i) {
+      const auto& field = handle.second->ref(static_cast<ProtoField>(i));
+      if (&sync_pb::EntitySpecifics::default_instance() != &field) {
+        entry_size += GetEntitySpecificsSize(field);

[stanisc, 2016/07/06 21:50:32]

Proto fields are often shared which means any consecutive fields with 
non-default values may deref to the same allocated EntitySpecifics object in
memory. It would be incorrect to add them twice (or 3 times).

[ssid, 2016/07/19 22:10:35]

Done.

+      }
+    }
+    for (unsigned i = ATTACHMENT_METADATA_FIELDS_BEGIN;
+         i < ATTACHMENT_METADATA_FIELDS_END; ++i) {
+      const auto& field =
+          handle.second->ref(static_cast<AttachmentMetadataField>(i));
+      if (&sync_pb::AttachmentMetadata::default_instance() != &field) {
+        entry_size += sizeof(sync_pb::AttachmentMetadata);

[stanisc, 2016/07/06 21:50:32]

The same is here.

[ssid, 2016/07/19 22:10:35]

Done.

+        entry_size +=
+            field.record_size() * (sizeof(sync_pb::AttachmentMetadataRecord) +
+                                   sizeof(sync_pb::AttachmentIdProto));
+        for (int i = 0; i < field.record_size(); ++i)
+          entry_size += field.record(i).id().unique_id().capacity();
+      }
+    }
+    for (unsigned i = UNIQUE_POSITION_FIELDS_BEGIN;
+         i < UNIQUE_POSITION_FIELDS_END; ++i) {
+      entry_size += handle.second->ref(static_cast<UniquePositionField>(i))
+                        .compressed_size();
+    }
+
+    handle.second->set_cached_size(entry_size);
+    total += entry_size;
+  }
+  return total;
+}
+
+void Directory::OnMemoryDump(base::trace_event::ProcessMemoryDump* pmd) {
+  auto dump = pmd->CreateAllocatorDump(base::StringPrintf(
+      "sync/0x%" PRIXPTR, reinterpret_cast<uintptr_t>(this)));
+  dump->AddScalar(base::trace_event::MemoryAllocatorDump::kNameSize,
+                  base::trace_event::MemoryAllocatorDump::kUnitsBytes,
+                  GetApproximateMemoryUsage());
+  const char* system_allocator_name =
+      base::trace_event::MemoryDumpManager::GetInstance()
+          ->system_allocator_pool_name();
+  if (system_allocator_name)
+    pmd->AddSuballocation(dump->guid(), system_allocator_name);
+}
+
 void Directory::SetDownloadProgress(
     ModelType model_type,
     const sync_pb::DataTypeProgressMarker& new_progress) {
   ScopedKernelLock lock(this);
   kernel_->persisted_info.download_progress[model_type].CopyFrom(new_progress);
   kernel_->info_status = KERNEL_SHARE_INFO_DIRTY;
 }
 
 bool Directory::HasEmptyDownloadProgress(ModelType type) const {
   ScopedKernelLock lock(this);
@@ skipping 646 matching lines @@
 Directory::Kernel* Directory::kernel() {
   return kernel_;
 }
 
 const Directory::Kernel* Directory::kernel() const {
   return kernel_;
 }
 
 }  // namespace syncable
 }  // namespace syncer