Harden allocator for file-backed memory.

base/metrics/persistent_memory_allocator.cc

 // Copyright (c) 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/metrics/persistent_memory_allocator.h"
 
 #include <assert.h>
 #include <algorithm>
 
 #if defined(OS_WIN)
 #include "winbase.h"
 #elif defined(OS_POSIX)
 #include <sys/mman.h>
 #endif
 
 #include "base/files/memory_mapped_file.h"
 #include "base/logging.h"
 #include "base/memory/shared_memory.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
+#include "base/threading/thread_restrictions.h"
 
 namespace {
 
 // Limit of memory segment size. It has to fit in an unsigned 32-bit number
 // and should be a power of 2 in order to accomodate almost any page size.
 const uint32_t kSegmentMaxSize = 1 << 30;  // 1 GiB
 
 // A constant (random) value placed in the shared metadata to identify
 // an already initialized memory segment.
 const uint32_t kGlobalCookie = 0x408305DC;
 
 // The current version of the metadata. If updates are made that change
 // the metadata, the version number can be queried to operate in a backward-
 // compatible manner until the memory segment is completely re-initalized.
-const uint32_t kGlobalVersion = 1;
+const uint32_t kGlobalVersion = 2;
 
 // Constant values placed in the block headers to indicate its state.
 const uint32_t kBlockCookieFree = 0;
 const uint32_t kBlockCookieQueue = 1;
 const uint32_t kBlockCookieWasted = (uint32_t)-1;
 const uint32_t kBlockCookieAllocated = 0xC8799269;
 
 // TODO(bcwhite): When acceptable, consider moving flags to std::atomic<char>
 // types rather than combined bitfield.
 
-// Flags stored in the flags_ field of the SharedMetaData structure below.
+// Flags stored in the flags_ field of the SharedMetadata structure below.
 enum : int {
   kFlagCorrupt = 1 << 0,
   kFlagFull    = 1 << 1
 };
 
 // Errors that are logged in "errors" histogram.
 enum AllocatorError : int {
   kMemoryIsCorrupt = 1,
 };
 
@@ skipping 36 matching lines @@
 // The block-header is placed at the top of every allocation within the
 // segment to describe the data that follows it.
 struct PersistentMemoryAllocator::BlockHeader {
   uint32_t size;       // Number of bytes in this block, including header.
   uint32_t cookie;     // Constant value indicating completed allocation.
   std::atomic<uint32_t> type_id;  // Arbitrary number indicating data type.
   std::atomic<uint32_t> next;     // Pointer to the next block when iterating.
 };
 
 // The shared metadata exists once at the top of the memory segment to
-// describe the state of the allocator to all processes.
+// describe the state of the allocator to all processes. The size of this
+// structure must be a multiple of 64-bits to ensure compatibility between
+// architectures.
 struct PersistentMemoryAllocator::SharedMetadata {
   uint32_t cookie;     // Some value that indicates complete initialization.
   uint32_t size;       // Total size of memory segment.
   uint32_t page_size;  // Paging size within memory segment.
   uint32_t version;    // Version code so upgrades don't break.
   uint64_t id;         // Arbitrary ID number given by creator.
   uint32_t name;       // Reference to stored name string.
+  uint32_t padding1;   // Pad-out read-only data to 64-bit alignment.
 
   // Above is read-only after first construction. Below may be changed and
   // so must be marked "volatile" to provide correct inter-process behavior.
 
+  // State of the memory, plus some padding to keep alignment.
+  volatile std::atomic<uint8_t> memory_state;  // MemoryState enum values.
+  uint8_t padding2[3];
+
   // Bitfield of information flags. Access to this should be done through
   // the CheckFlag() and SetFlag() methods defined above.
   volatile std::atomic<uint32_t> flags;
 
   // Offset/reference to first free space in segment.
   volatile std::atomic<uint32_t> freeptr;
 
   // The "iterable" queue is an M&S Queue as described here, append-only:
   // https://www.research.ibm.com/people/m/michael/podc-1996.pdf
+  // |queue| needs to be 64-bit aligned and is itself a multiple of 64 bits.
   volatile std::atomic<uint32_t> tailptr;  // Last block of iteration queue.
   volatile BlockHeader queue;   // Empty block for linked-list head/tail.
 };
 
 // The "queue" block header is used to detect "last node" so that zero/null
 // can be used to indicate that it hasn't been added at all. It is part of
 // the SharedMetadata structure which itself is always located at offset zero.
 const PersistentMemoryAllocator::Reference
     PersistentMemoryAllocator::kReferenceQueue =
         offsetof(SharedMetadata, queue);
@@ skipping 171 matching lines @@
       readonly_(readonly),
       corrupt_(0),
       allocs_histogram_(nullptr),
       used_histogram_(nullptr),
       errors_histogram_(nullptr) {
   // These asserts ensure that the structures are 32/64-bit agnostic and meet
   // all the requirements of use within the allocator. They access private
   // definitions and so cannot be moved to the global scope.
   static_assert(sizeof(PersistentMemoryAllocator::BlockHeader) == 16,
                 "struct is not portable across different natural word widths");
-  static_assert(sizeof(PersistentMemoryAllocator::SharedMetadata) == 56,
+  static_assert(sizeof(PersistentMemoryAllocator::SharedMetadata) == 64,
                 "struct is not portable across different natural word widths");
 
   static_assert(sizeof(BlockHeader) % kAllocAlignment == 0,
                 "BlockHeader is not a multiple of kAllocAlignment");
   static_assert(sizeof(SharedMetadata) % kAllocAlignment == 0,
                 "SharedMetadata is not a multiple of kAllocAlignment");
   static_assert(kReferenceQueue % kAllocAlignment == 0,
                 "\"queue\" is not aligned properly; must be at end of struct");
 
   // Ensure that memory segment is of acceptable size.
@@ skipping 51 matching lines @@
     shared_meta()->tailptr.store(kReferenceQueue, std::memory_order_release);
 
     // Allocate space for the name so other processes can learn it.
     if (!name.empty()) {
       const size_t name_length = name.length() + 1;
       shared_meta()->name = Allocate(name_length, 0);
       char* name_cstr = GetAsArray<char>(shared_meta()->name, 0, name_length);
       if (name_cstr)
         memcpy(name_cstr, name.data(), name.length());
     }
+
+    shared_meta()->memory_state.store(MEMORY_INITIALIZED,
+                                      std::memory_order_release);
   } else {
-    if (shared_meta()->size == 0 ||
-        shared_meta()->version == 0 ||
+    if (shared_meta()->size == 0 || shared_meta()->version != kGlobalVersion ||
         shared_meta()->freeptr.load(std::memory_order_relaxed) == 0 ||
-        shared_meta()->tailptr == 0 ||
-        shared_meta()->queue.cookie == 0 ||
+        shared_meta()->tailptr == 0 || shared_meta()->queue.cookie == 0 ||
         shared_meta()->queue.next.load(std::memory_order_relaxed) == 0) {
       SetCorrupt();
     }
     if (!readonly) {
       // The allocator is attaching to a previously initialized segment of
       // memory. If the initialization parameters differ, make the best of it
       // by reducing the local construction parameters to match those of
       // the actual memory area. This ensures that the local object never
       // tries to write outside of the original bounds.
       // Because the fields are const to ensure that no code other than the
@@ skipping 60 matching lines @@
   used_histogram_ = LinearHistogram::FactoryGet(
       "UMA.PersistentAllocator." + name_string + ".UsedPct", 1, 101, 21,
       HistogramBase::kUmaTargetedHistogramFlag);
 
   DCHECK(!errors_histogram_);
   errors_histogram_ = SparseHistogram::FactoryGet(
       "UMA.PersistentAllocator." + name_string + ".Errors",
       HistogramBase::kUmaTargetedHistogramFlag);
 }
 
+void PersistentMemoryAllocator::Flush(bool sync) {
+  FlushPartial(used(), sync);
+}
+
+void PersistentMemoryAllocator::SetMemoryState(uint8_t memory_state) {
+  shared_meta()->memory_state.store(memory_state, std::memory_order_relaxed);
+  FlushPartial(sizeof(SharedMetadata), false);
+}
+
+uint8_t PersistentMemoryAllocator::GetMemoryState() const {
+  return shared_meta()->memory_state.load(std::memory_order_relaxed);
+}
+
 size_t PersistentMemoryAllocator::used() const {
   return std::min(shared_meta()->freeptr.load(std::memory_order_relaxed),
                   mem_size_);
 }
 
 PersistentMemoryAllocator::Reference PersistentMemoryAllocator::GetAsReference(
     const void* memory,
     uint32_t type_id) const {
   uintptr_t address = reinterpret_cast<uintptr_t>(memory);
   if (address < reinterpret_cast<uintptr_t>(mem_base_))
@@ skipping 317 matching lines @@
 
 // Dereference a block |ref| and ensure that it's valid for the desired
 // |type_id| and |size|. |special| indicates that we may try to access block
 // headers not available to callers but still accessed by this module. By
 // having internal dereferences go through this same function, the allocator
 // is hardened against corruption.
 const volatile PersistentMemoryAllocator::BlockHeader*
 PersistentMemoryAllocator::GetBlock(Reference ref, uint32_t type_id,
                                     uint32_t size, bool queue_ok,
                                     bool free_ok) const {
+  // Handle special cases.
+  if (ref == kReferenceQueue && queue_ok)
+    return reinterpret_cast<const volatile BlockHeader*>(mem_base_ + ref);
+
   // Validation of parameters.
-  if (ref < (queue_ok ? kReferenceQueue : sizeof(SharedMetadata)))
+  if (ref < sizeof(SharedMetadata))
     return nullptr;
   if (ref % kAllocAlignment != 0)
     return nullptr;
   size += sizeof(BlockHeader);
   if (ref + size > mem_size_)
     return nullptr;
 
   // Validation of referenced block-header.
   if (!free_ok) {
-    uint32_t freeptr = std::min(
-        shared_meta()->freeptr.load(std::memory_order_relaxed), mem_size_);
-    if (ref + size > freeptr)
-      return nullptr;
     const volatile BlockHeader* const block =
         reinterpret_cast<volatile BlockHeader*>(mem_base_ + ref);
+    if (block->cookie != kBlockCookieAllocated)
+      return nullptr;
     if (block->size < size)
       return nullptr;
-    if (ref + block->size > freeptr)
-      return nullptr;
-    if (ref != kReferenceQueue && block->cookie != kBlockCookieAllocated)
+    if (ref + block->size > mem_size_)
       return nullptr;
     if (type_id != 0 &&
         block->type_id.load(std::memory_order_relaxed) != type_id) {
       return nullptr;
     }
   }
 
   // Return pointer to block data.
   return reinterpret_cast<const volatile BlockHeader*>(mem_base_ + ref);
 }
 
+void PersistentMemoryAllocator::FlushPartial(size_t length, bool sync) {
+  // Generally there is nothing to do as every write is done through volatile
+  // memory with atomic instructions to guarantee consistency. This (virtual)
+  // method exists so that derivced classes can do special things, such as
+  // tell the OS to write changes to disk now rather than when convenient.
+}
+
 void PersistentMemoryAllocator::RecordError(int error) const {
   if (errors_histogram_)
     errors_histogram_->Add(error);
 }
 
 const volatile void* PersistentMemoryAllocator::GetBlockData(
     Reference ref,
     uint32_t type_id,
     uint32_t size) const {
   DCHECK(size > 0);
@@ skipping 120 matching lines @@
     uint64_t id,
     base::StringPiece name,
     bool read_only)
     : PersistentMemoryAllocator(
           Memory(const_cast<uint8_t*>(file->data()), MEM_FILE),
           max_size != 0 ? max_size : file->length(),
           0,
           id,
           name,
           read_only),
-      mapped_file_(std::move(file)) {}
+      mapped_file_(std::move(file)) {
+  // Ensure the disk-copy of the data reflects the fully-initialized memory as
+  // there is no guarantee as to what order the pages might be auto-flushed by
+  // the OS in the future.
+  Flush(true);
+}
 
 FilePersistentMemoryAllocator::~FilePersistentMemoryAllocator() {}
 
 // static
 bool FilePersistentMemoryAllocator::IsFileAcceptable(
     const MemoryMappedFile& file,
     bool read_only) {
   return IsMemoryAcceptable(file.data(), file.length(), 0, read_only);
 }
+
+void FilePersistentMemoryAllocator::FlushPartial(size_t length, bool sync) {
+  if (sync)
+    ThreadRestrictions::AssertIOAllowed();
+  if (IsReadonly())
+    return;
+
+#if defined(OS_WIN)
+  // Windows doesn't support a synchronous flush.
+  BOOL success = ::FlushViewOfFile(data(), length);
+  DPCHECK(success);
+#elif defined(OS_MACOSX)
+  // On OSX, "invalidate" removes all cached pages, forcing a re-read from
+  // disk. That's not applicable to "flush" so omit it.
+  int result =
+      ::msync(const_cast<void*>(data()), length, sync ? MS_SYNC : MS_ASYNC);
+  DCHECK_NE(EINVAL, result);
+#elif defined(OS_POSIX)
+  // On POSIX, "invalidate" forces _other_ processes to recognize what has
+  // been written to disk and so is applicable to "flush".
+  int result = ::msync(const_cast<void*>(data()), length,
+                       MS_INVALIDATE | (sync ? MS_SYNC : MS_ASYNC));
+  DCHECK_NE(EINVAL, result);
+#else
+#error Unsupported OS.
+#endif
+}
 #endif  // !defined(OS_NACL)
 
 }  // namespace base