Add tools/android/memdump/.

tools/android/memdump/memdump.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <fcntl.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "base/basictypes.h"
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/file_util.h"
+#include "base/hash_tables.h"
+#include "base/logging.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/string_number_conversions.h"
+#include "base/stringprintf.h"
+#include "base/strings/string_piece.h"
+#include "base/strings/string_split.h"
+
+namespace {
+
+// An entry in /proc/<pid>/pagemap.
+struct PageMapEntry {
+  uint64 page_frame_number : 55;
+  uint unused : 8;
+  uint present : 1;
+};
+
+// Describes a virtual page.
+struct PageInfo {
+  int64 page_frame_number : 55;  // Physical page id, also known as PFN.
+  int64 times_mapped : 9 + 32;
+};
+
+struct MemoryMap {
+  std::string name;
+  std::string flags;
+  uint start_address;
+  uint end_address;
+  int private_count;
+  int app_shared_count;
+  int other_shared_count;
+  std::vector<PageInfo> committed_pages;
+};
+
+struct ProcessMemory {
+  pid_t pid;
+  std::vector<MemoryMap> memory_maps;
+};
+
+// Number of times a physical page is mapped in a process.
+typedef base::hash_map<int64, int> PFNMap;
+
+// Parses lines from /proc/<PID>/maps, e.g.:
+// 401e7000-401f5000 r-xp 00000000 103:02 158       /system/bin/linker
+bool ParseMemoryMapLine(const std::string& line,
+                        std::vector<std::string>* tokens,
+                        MemoryMap* memory_map) {
+  static const char kExample[] = "7fe94ae7d000-7fe94bbd0000 r--p";

[bulach, 2013/05/30 11:47:07]

nit: instead of strlen below, perhaps:
static const int kExampleLen = sizeof(kExample) - 1;

[Philippe, 2013/05/30 13:48:43]

Done. I got rid of this and addressed when I addressed some comments that
Benjamin made offline.

+  do {
+    tokens->clear();
+    if (line.length() < strlen(kExample))
+      break;
+    base::SplitString(line, ' ', tokens);
+    if (tokens->size() < 2)
+      break;
+    const std::string& addr_range = tokens->at(0);
+    int64 tmp;
+    if (!base::HexStringToInt64(
+            base::StringPiece(addr_range.begin(), addr_range.begin() + 8),
+            &tmp)) {
+      break;
+    }
+    memory_map->start_address = static_cast<uint>(tmp);
+    if (!base::HexStringToInt64(
+            base::StringPiece(addr_range.begin() + 9,
+                              addr_range.begin() + 9 + 8), &tmp)) {
+      break;
+    }
+    memory_map->end_address = static_cast<uint>(tmp);
+    const int kModeLength = strlen("rwxp");
+    if (tokens->at(1).size() < kModeLength)
+      break;
+    memory_map->flags.swap(tokens->at(1));
+    if (tokens->size() >= 6) {
+      for (std::vector<std::string>::const_iterator it = tokens->begin() + 5;
+           it != tokens->end(); ++it) {
+        if (!it->empty()) {
+          if (!memory_map->name.empty())
+            memory_map->name.append(" ");
+          memory_map->name.append(*it);
+        }
+      }
+    }
+    return true;
+  } while (false);

[bulach, 2013/05/30 11:47:07]

nit: perhaps split into a TokenizeMemoryMapLine() that would return false
instead of break the loop, then here something like:
if (TokenizeMemoryMapLine(..))
  return true;
LOG(ERROR) ....
return false;

[Philippe, 2013/05/30 13:48:43]

Good idea. I just moved the error message to the caller site.

+
+  LOG(ERROR) << "Could not parse line: " << line;
+  return false;
+}
+
+// Reads sizeof(*value) bytes from file |fd| at |offset|.

[bulach, 2013/05/30 11:47:07]

sizeof(T), and in 119 as well?

[Philippe, 2013/05/30 13:48:43]

I made the change here but kept line 119 unchanged if that's ok with you. I find
it safer to use sizeof(*value) since I'm directly writing to value.

+template <typename T>
+bool ReadFromFileAtOffset(int fd, off_t offset, T* value) {

[qsr, 2013/05/30 10:30:25]

You might want to use mmap instead of opening your /proc files. You will need to
check if that wokr for your particular files though.

[digit, 2013/05/30 10:34:09]

From past experience, most files in /proc cannot be mmapped properly :-)

+  if (lseek64(fd, offset * sizeof(*value), SEEK_SET) < 0) {
+    PLOG(ERROR) << "lseek";
+    return false;
+  }
+  ssize_t bytes = read(fd, value, sizeof(*value));
+  if (bytes < 0) {

[qsr, 2013/05/30 10:30:25]

You could replace 0 by sizeof(*value) if you do not expect partial reads.

[Philippe, 2013/05/30 13:48:43]

Done.

+    PLOG(ERROR) << "read";
+    return false;
+  }
+  return true;
+}
+
+// Fills |process_maps| in with the process memory maps identified by |pid|.
+bool GetProcessMaps(pid_t pid, std::vector<MemoryMap>* process_maps) {
+  std::ifstream maps_file(base::StringPrintf("/proc/%d/maps", pid).c_str());
+  if (!maps_file.good()) {
+    PLOG(ERROR) << "open";
+    return false;
+  }
+  std::string line;
+  std::vector<std::string> tokens;
+  while (std::getline(maps_file, line) && !line.empty()) {
+    MemoryMap memory_map = {};
+    if (!ParseMemoryMapLine(line, &tokens, &memory_map))
+      return false;
+    process_maps->push_back(memory_map);
+  }
+  return true;
+}
+
+// Fills |committed_pages| in with the set of committed pages contained in the
+// provided memory map.
+bool GetPagesForMemoryMap(int pagemap_fd,
+                          const MemoryMap& memory_map,
+                          std::vector<PageInfo>* committed_pages) {
+  for (uint addr = memory_map.start_address; addr < memory_map.end_address;
+       addr += PAGE_SIZE) {
+    DCHECK_EQ(0, addr % PAGE_SIZE);
+    PageMapEntry page_map_entry = {};
+    COMPILE_ASSERT(sizeof(PageMapEntry) == sizeof(uint64), unexpected_size);
+    const off64_t offset = addr / PAGE_SIZE;
+    if (!ReadFromFileAtOffset(pagemap_fd, offset, &page_map_entry))
+      return false;
+    if (page_map_entry.present) {  // Ignore non-committed pages.
+      if (page_map_entry.page_frame_number == 0)
+        continue;
+      PageInfo page_info = {};
+      page_info.page_frame_number = page_map_entry.page_frame_number;
+      committed_pages->push_back(page_info);
+    }
+  }
+  return true;
+}
+
+bool SetTimesMapped(int pagecount_fd, std::vector<PageInfo>* pages) {
+  for (std::vector<PageInfo>::iterator it = pages->begin();
+       it != pages->end(); ++it) {
+    PageInfo* const page_info = &*it;
+    int64 times_mapped;
+    if (!ReadFromFileAtOffset(
+            pagecount_fd, page_info->page_frame_number, &times_mapped)) {
+      return false;
+    }
+    page_info->times_mapped = times_mapped;
+  }
+  return true;
+}
+
+// Fills in the provided vector of Page Frame Number maps. This lets
+// ClassifyPages() know how many times each page is mapped in the processes.
+void FillPFNMaps(const std::vector<ProcessMemory>& processes_memory,
+                 std::vector<PFNMap>* pfn_maps) {
+  int current_process_index = 0;
+  for (std::vector<ProcessMemory>::const_iterator it = processes_memory.begin();
+       it != processes_memory.end(); ++it, ++current_process_index) {
+    const std::vector<MemoryMap>& memory_maps = it->memory_maps;
+    for (std::vector<MemoryMap>::const_iterator it = memory_maps.begin();
+         it != memory_maps.end(); ++it) {
+      const std::vector<PageInfo>& pages = it->committed_pages;
+      for (std::vector<PageInfo>::const_iterator it = pages.begin();
+           it != pages.end(); ++it) {
+        const PageInfo& page_info = *it;
+        PFNMap* const pfn_map = &(*pfn_maps)[current_process_index];
+        const std::pair<PFNMap::iterator, bool> result = pfn_map->insert(
+            std::make_pair(page_info.page_frame_number, 0));
+        ++result.first->second;
+      }
+    }
+  }
+}
+
+// Sets the private_count/app_shared_count/other_shared_count fields of the
+// provided memory maps for each process.
+void ClassifyPages(std::vector<ProcessMemory>* processes_memory) {
+  std::vector<PFNMap> pfn_maps(processes_memory->size());
+  FillPFNMaps(*processes_memory, &pfn_maps);
+  for (std::vector<ProcessMemory>::iterator it = processes_memory->begin();
+       it != processes_memory->end(); ++it) {
+    std::vector<MemoryMap>* const memory_maps = &it->memory_maps;
+    for (std::vector<MemoryMap>::iterator it = memory_maps->begin();
+         it != memory_maps->end(); ++it) {
+      MemoryMap* const memory_map = &*it;
+      const std::vector<PageInfo>& pages = memory_map->committed_pages;
+      for (std::vector<PageInfo>::const_iterator it = pages.begin();
+           it != pages.end(); ++it) {
+        const PageInfo& page_info = *it;
+        if (page_info.times_mapped == 1) {
+          ++memory_map->private_count;
+          continue;
+        }
+        bool mapped_in_multiple_processes = false;
+        int times_mapped = 0;
+        // See if the current physical page is also mapped in the processes that
+        // are being analyzed.
+        for (std::vector<PFNMap>::const_iterator it = pfn_maps.begin();
+             it != pfn_maps.end(); ++it) {
+          const PFNMap& pfn_map = *it;
+          const PFNMap::const_iterator found_it = pfn_map.find(
+              page_info.page_frame_number);
+          if (found_it == pfn_map.end())
+            continue;
+          if (times_mapped)
+            mapped_in_multiple_processes = true;
+          times_mapped += found_it->second;
+        }
+        if (times_mapped == page_info.times_mapped) {
+          // The physical page is only mapped in the processes that are being
+          // analyzed.
+          if (mapped_in_multiple_processes)
+            ++memory_map->app_shared_count;
+          else  // Mapped multiple times in the same process.
+            ++memory_map->private_count;
+        } else {
+          ++memory_map->other_shared_count;
+        }
+      }
+    }
+  }
+}
+
+void DumpProcessesMemoryMaps(
+    const std::vector<ProcessMemory>& processes_memory) {
+  std::string buf;
+  for (std::vector<ProcessMemory>::const_iterator it = processes_memory.begin();
+       it != processes_memory.end(); ++it) {
+    const ProcessMemory& process_memory = *it;
+    std::cout << "[ PID=" << process_memory.pid << "]" << '\n';
+    const std::vector<MemoryMap>& memory_maps = process_memory.memory_maps;
+    for (std::vector<MemoryMap>::const_iterator it = memory_maps.begin();
+         it != memory_maps.end(); ++it) {
+      const MemoryMap& memory_map = *it;
+      base::SStringPrintf(
+          &buf, "%x-%x %s %d %d %d %s\n", memory_map.start_address,
+          memory_map.end_address, memory_map.flags.c_str(),
+          memory_map.private_count * PAGE_SIZE,
+          memory_map.app_shared_count * PAGE_SIZE,
+          memory_map.other_shared_count * PAGE_SIZE,
+          memory_map.name.c_str());
+      std::cout << buf;
+    }
+  }
+}
+
+void ResumeProcesses(const std::vector<pid_t>& pids) {
+  for (std::vector<pid_t>::const_iterator it = pids.begin(); it != pids.end();
+       ++it) {
+    kill(*it, SIGCONT);
+  }
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  if (argc == 1) {
+    LOG(ERROR) << "Usage: " << argv[0] << " <PID1>... <PIDN>";
+    return EXIT_FAILURE;
+  }
+  std::vector<pid_t> pids;
+  for (const char* const* ptr = argv + 1; *ptr; ++ptr) {
+    pid_t pid;
+    if (!base::StringToInt(*ptr, &pid))
+      return EXIT_FAILURE;
+    pids.push_back(pid);
+  }
+  int page_count_fd = open("/proc/kpagecount", O_RDONLY);
+  if (page_count_fd < 0) {
+    PLOG(ERROR) << "open";
+    return EXIT_FAILURE;
+  }
+  file_util::ScopedFD page_count_fd_closer(&page_count_fd);
+
+  base::ScopedClosureRunner auto_resume_processes(
+      base::Bind(&ResumeProcesses, pids));

[bulach, 2013/05/30 11:47:07]

s/ResumeProcess/KillAll/, pass the signal in, and remove 304-307 below

[Philippe, 2013/05/30 13:48:43]

Great idea! I should have thought about it.

+  std::vector<ProcessMemory> processes_memory(pids.size());
+  for (std::vector<pid_t>::const_iterator it = pids.begin(); it != pids.end();
+       ++it) {
+    kill(*it, SIGSTOP);
+  }
+  for (std::vector<pid_t>::const_iterator it = pids.begin(); it != pids.end();
+       ++it) {
+    const pid_t pid = *it;

[qsr, 2013/05/30 10:30:26]

You could extract this content in another method.

[Philippe, 2013/05/30 13:48:43]

Done.

+    int pagemap_fd = open(
+        base::StringPrintf("/proc/%d/pagemap", pid).c_str(), O_RDONLY);
+    if (pagemap_fd < 0) {
+      PLOG(ERROR) << "open";
+      return EXIT_FAILURE;
+    }
+    file_util::ScopedFD auto_closer(&pagemap_fd);
+    ProcessMemory* const process_memory = &processes_memory[it - pids.begin()];
+    process_memory->pid = pid;
+    std::vector<MemoryMap>* const process_maps = &process_memory->memory_maps;
+    if (!GetProcessMaps(pid, process_maps))
+      return EXIT_FAILURE;
+    for (std::vector<MemoryMap>::iterator it = process_maps->begin();
+         it != process_maps->end(); ++it) {
+      std::vector<PageInfo>* const committed_pages =
+          &it->committed_pages;
+      GetPagesForMemoryMap(pagemap_fd, *it, committed_pages);
+      SetTimesMapped(page_count_fd, committed_pages);
+    }
+  }
+  ResumeProcesses(pids);
+  auto_resume_processes.Release();

[qsr, 2013/05/30 10:30:26]

instead of this, you could have it in a scope, and just let the scope ends, that
would also close page_count_fd

[bulach, 2013/05/30 11:47:07]

or parhaps:
delete 332, and here:
auto_resume_processes.Release().Run();

[Philippe, 2013/05/30 13:48:43]

I followed Benjamin's suggestion. To me it looks less Chromium idiomatic (than
your suggestion Marcus) but might be slightly easier to read.

+  ClassifyPages(&processes_memory);
+  DumpProcessesMemoryMaps(processes_memory);
+  return EXIT_SUCCESS;
+}