Adding extended report to memdump with resident pages bit reporting.

tools/android/memdump/memdump.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <fcntl.h>
 #include <signal.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <algorithm>
 #include <cstring>
 #include <fstream>
 #include <iostream>
 #include <limits>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include "base/base64.h"
 #include "base/basictypes.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/containers/hash_tables.h"
 #include "base/file_util.h"
 #include "base/logging.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_piece.h"
 #include "base/strings/string_split.h"
 #include "base/strings/stringprintf.h"
 
 namespace {
 
+class BitSet {
+ public:
+  BitSet() : data_(0) {}

[pliard, 2013/07/23 09:08:22]

This will use the std::vector<char>::vector<char>(size_t size) constructor (note
the |size| as opposed to |capacity|). So this has the same effect as the default
constructor. C++'s philosophy is not to pay for what you don't use so I doubt
the default constructor (with size 0) is pre-allocating a dynamic buffer. Not
sure this is a good argument :)

[Primiano Tucci (use gerrit), 2013/07/23 22:00:17]

Well, the default constructor without the argument is slightly different (
vector(const allocator_type& alloc = allocator_type()). Btw, it seems that the
inizial capacity, in both cases, is 0. So, agree with you, let's keep it clean.

+
+  void resize(size_t nbits) {
+    data_.resize((nbits + 7) / 8);
+  }
+
+  void set(uint32 bit) {
+    const uint32 byte_idx = bit / 8;
+    CHECK(byte_idx < data_.size());
+    data_[byte_idx] |= (1 << (bit & 7));
+  }
+
+  const std::string AsB64String() const {

[pliard, 2013/07/23 09:08:22]

Nit: this 'const' is superfluous since you are returning a copy. Note that with
your signature the caller could still write 'std::string s =
bit_set.AsB64String()'. This would cause yet another copy :)

[Primiano Tucci (use gerrit), 2013/07/23 22:00:17]

Yeah, that was a leftover from a previous attempt (to use directly a string as
back storage and return that one w/ encoding).
Done!

+    std::string bits(&data_[0], data_.size());
+    std::string b64_string;
+    base::Base64Encode(bits, &b64_string);
+    return b64_string;
+  }
+
+ private:
+  std::vector<char> data_;
+};
+
 // An entry in /proc/<pid>/pagemap.
 struct PageMapEntry {
   uint64 page_frame_number : 55;
   uint unused : 8;
   uint present : 1;
 };
 
 // Describes a memory page.
 struct PageInfo {
   int64 page_frame_number; // Physical page id, also known as PFN.
   int64 flags;
   int32 times_mapped;
 };
 
 struct MemoryMap {
   std::string name;
   std::string flags;
   uint start_address;
   uint end_address;
+  uint offset;
   int private_count;
   int unevictable_private_count;
   int other_shared_count;
   int unevictable_other_shared_count;
   // app_shared_counts[i] contains the number of pages mapped in i+2 processes
   // (only among the processes that are being analyzed).
   std::vector<int> app_shared_counts;
   std::vector<PageInfo> committed_pages;
+  // committed_pages_bits is a bitset reflecting the present bit for all the
+  // virtual pages of the mapping.
+  BitSet committed_pages_bits;
 };
 
 struct ProcessMemory {
   pid_t pid;
   std::vector<MemoryMap> memory_maps;
 };
 
 bool PageIsUnevictable(const PageInfo& page_info) {
   // These constants are taken from kernel-page-flags.h.
   const int KPF_DIRTY = 4; // Note that only file-mapped pages can be DIRTY.
@@ skipping 37 matching lines @@
           base::StringPiece(
               addr_range.begin() + end_addr_start_pos,
               addr_range.begin() + end_addr_start_pos + addr_len),
           &tmp)) {
     return false;
   }
   memory_map->end_address = static_cast<uint>(tmp);
   if (tokens->at(1).size() != strlen("rwxp"))
     return false;
   memory_map->flags.swap(tokens->at(1));
+  if (!base::HexStringToUInt64(tokens->at(2), &tmp))
+    return false;
+  memory_map->offset = static_cast<uint>(tmp);
+  memory_map->committed_pages_bits.resize(
+      (memory_map->end_address - memory_map->start_address) / PAGE_SIZE);
   const int map_name_index = 5;
   if (tokens->size() >= map_name_index + 1) {
     for (std::vector<std::string>::const_iterator it =
              tokens->begin() + map_name_index; it != tokens->end(); ++it) {
       if (!it->empty()) {
         if (!memory_map->name.empty())
           memory_map->name.append(" ");
         memory_map->name.append(*it);
       }
     }
@@ skipping 33 matching lines @@
     }
     process_maps->push_back(memory_map);
   }
   return true;
 }
 
 // Fills |committed_pages| in with the set of committed pages contained in the
 // provided memory map.
 bool GetPagesForMemoryMap(int pagemap_fd,
                           const MemoryMap& memory_map,
-                          std::vector<PageInfo>* committed_pages) {
-  for (uint addr = memory_map.start_address; addr < memory_map.end_address;
-       addr += PAGE_SIZE) {
+                          std::vector<PageInfo>* committed_pages,
+                          BitSet* committed_pages_bits) {
+  for (uint addr = memory_map.start_address, page_index = 0;
+       addr < memory_map.end_address;
+       addr += PAGE_SIZE, ++page_index) {
     DCHECK_EQ(0, addr % PAGE_SIZE);
     PageMapEntry page_map_entry = {};
     COMPILE_ASSERT(sizeof(PageMapEntry) == sizeof(uint64), unexpected_size);
     const off64_t offset = addr / PAGE_SIZE;
     if (!ReadFromFileAtOffset(pagemap_fd, offset, &page_map_entry))
       return false;
     if (page_map_entry.present) {  // Ignore non-committed pages.
       if (page_map_entry.page_frame_number == 0)
         continue;
       PageInfo page_info = {};
       page_info.page_frame_number = page_map_entry.page_frame_number;
       committed_pages->push_back(page_info);
+      committed_pages_bits->set(page_index);
     }
   }
   return true;
 }
 
 // Fills |committed_pages| with mapping count and flags information gathered
 // looking-up /proc/kpagecount and /proc/kpageflags.
 bool SetPagesInfo(int pagecount_fd,
                   int pageflags_fd,
                   std::vector<PageInfo>* pages) {
@@ skipping 185 matching lines @@
     base::SStringPrintf(
         &buf, "%d\t%d\t\t%d\t\t%d\n",
         process_memory.pid,
         total_private * KB_PER_PAGE,
         static_cast<int>(total_app_shared) * KB_PER_PAGE,
         total_other_shared * KB_PER_PAGE);
     std::cout << buf;
   }
 }
 
+void DumpProcessesMemoryMapsInExtendedFormat(
+    const std::vector<ProcessMemory>& processes_memory) {
+  std::string buf;
+  std::string app_shared_buf;
+  for (std::vector<ProcessMemory>::const_iterator it = processes_memory.begin();
+       it != processes_memory.end(); ++it) {
+    const ProcessMemory& process_memory = *it;
+    std::cout << "[ PID=" << process_memory.pid << "]" << '\n';
+    const std::vector<MemoryMap>& memory_maps = process_memory.memory_maps;
+    for (std::vector<MemoryMap>::const_iterator it = memory_maps.begin();
+         it != memory_maps.end(); ++it) {
+      const MemoryMap& memory_map = *it;
+      app_shared_buf.clear();
+      AppendAppSharedField(memory_map.app_shared_counts, &app_shared_buf);
+      base::SStringPrintf(
+          &buf,
+          "%x-%x %s %x private_unevictable=%d private=%d shared_app=%s "
+          "shared_other_unevictable=%d shared_other=%d \"%s\" [%s]\n",
+          memory_map.start_address,
+          memory_map.end_address,
+          memory_map.flags.c_str(),
+          memory_map.offset,
+          memory_map.unevictable_private_count * PAGE_SIZE,
+          memory_map.private_count * PAGE_SIZE,
+          app_shared_buf.c_str(),
+          memory_map.unevictable_other_shared_count * PAGE_SIZE,
+          memory_map.other_shared_count * PAGE_SIZE,
+          memory_map.name.c_str(),
+          memory_map.committed_pages_bits.AsB64String().c_str());
+      std::cout << buf;
+    }
+  }
+}
+
 bool CollectProcessMemoryInformation(int page_count_fd,
                                      int page_flags_fd,
                                      ProcessMemory* process_memory) {
   const pid_t pid = process_memory->pid;
   int pagemap_fd = open(
       base::StringPrintf("/proc/%d/pagemap", pid).c_str(), O_RDONLY);
   if (pagemap_fd < 0) {
     PLOG(ERROR) << "open";
     return false;
   }
   file_util::ScopedFD auto_closer(&pagemap_fd);
   std::vector<MemoryMap>* const process_maps = &process_memory->memory_maps;
   if (!GetProcessMaps(pid, process_maps))
     return false;
   for (std::vector<MemoryMap>::iterator it = process_maps->begin();
        it != process_maps->end(); ++it) {
     std::vector<PageInfo>* const committed_pages = &it->committed_pages;
-    GetPagesForMemoryMap(pagemap_fd, *it, committed_pages);
+    BitSet* const pages_bits = &it->committed_pages_bits;
+    GetPagesForMemoryMap(pagemap_fd, *it, committed_pages, pages_bits);
     SetPagesInfo(page_count_fd, page_flags_fd, committed_pages);
   }
   return true;
 }
 
 void KillAll(const std::vector<pid_t>& pids, int signal_number) {
   for (std::vector<pid_t>::const_iterator it = pids.begin(); it != pids.end();
        ++it) {
     kill(*it, signal_number);
   }
 }
 
 }  // namespace
 
 int main(int argc, char** argv) {
-  bool short_output = false;
   if (argc == 1) {
-    LOG(ERROR) << "Usage: " << argv[0] << " [-a] <PID1>... <PIDN>";
+    LOG(ERROR) << "Usage: " << argv[0] << " [-a|-x] <PID1>... <PIDN>";
     return EXIT_FAILURE;
   }
-  if (!strncmp(argv[1], "-a", 2)) {
+  const bool short_output = !strncmp(argv[1], "-a", 2);
+  const bool extended_output = !strncmp(argv[1], "-x", 2);
+  if (short_output || extended_output) {
     if (argc == 2) {
-      LOG(ERROR) << "Usage: " << argv[0] << " [-a] <PID1>... <PIDN>";
+      LOG(ERROR) << "Usage: " << argv[0] << " [-a|-x] <PID1>... <PIDN>";
       return EXIT_FAILURE;
     }
-    short_output = true;
     ++argv;
   }
   std::vector<pid_t> pids;
   for (const char* const* ptr = argv + 1; *ptr; ++ptr) {
     pid_t pid;
     if (!base::StringToInt(*ptr, &pid))
       return EXIT_FAILURE;
     pids.push_back(pid);
   }
 
@@ skipping 25 matching lines @@
       if (!CollectProcessMemoryInformation(page_count_fd,
                                            page_flags_fd,
                                            process_memory))
         return EXIT_FAILURE;
     }
   }
 
   ClassifyPages(&processes_memory);
   if (short_output)
     DumpProcessesMemoryMapsInShortFormat(processes_memory);
+  else if (extended_output)
+    DumpProcessesMemoryMapsInExtendedFormat(processes_memory);
   else
     DumpProcessesMemoryMaps(processes_memory);
   return EXIT_SUCCESS;
 }