Allocator shims design prototype.

syzygy/integration_tests/allocator_shim.cc

Index: syzygy/integration_tests/allocator_shim.cc
diff --git a/syzygy/integration_tests/allocator_shim.cc b/syzygy/integration_tests/allocator_shim.cc
new file mode 100644
index 0000000000000000000000000000000000000000..44168b6d209d2ee76456434f47649f9b8481a866
--- /dev/null
+++ b/syzygy/integration_tests/allocator_shim.cc
@@ -0,0 +1,117 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file defines symbols to override the same functions in the Visual C++
+// CRT implementation.
+
+#include <windows.h>
+#include <malloc.h>
+
+extern "C" {
+
+namespace {
+
+// Definitions of the following heap management functions provided by
+// the SyzyAsan library:
+// heap_create, heap_destroy, heap_alloc, heap_realloc, and heap_free
+typedef HANDLE(WINAPI* HeapCreatePtr)(DWORD, SIZE_T, SIZE_T);
+typedef BOOL(WINAPI* HeapDestroyPtr)(HANDLE);
+typedef LPVOID(WINAPI* HeapAllocPtr)(HANDLE, DWORD, SIZE_T);
+typedef LPVOID(WINAPI* HeapReAllocPtr)(HANDLE, DWORD, LPVOID, SIZE_T);
+typedef BOOL(WINAPI* HeapFreePtr)(HANDLE, DWORD, LPVOID);
+typedef BOOL(WINAPI* HeapDestroyPtr)(HANDLE);
+struct AsanRuntimePointers {
+  HANDLE asan_heap = nullptr;
+  HMODULE asan_module = nullptr;
+  HeapCreatePtr heap_create = nullptr;
+  HeapAllocPtr heap_alloc = nullptr;
+  HeapFreePtr heap_free = nullptr;
+  HeapReAllocPtr heap_realloc = nullptr;
+  HeapDestroyPtr heap_destroy = nullptr;
+  AsanRuntimePointers(){};
+};
+
+// It retreives the handle for the syzyasan_rtl.dll module

[Sigurður Ásgeirsson, 2017/07/31 19:05:52]

nit: speling [sic]

[njanevsk, 2017/08/01 15:46:43]

Done.

+// and the heap functions that it provides.
+void ResolveAsanRuntimePointers(AsanRuntimePointers* asan_runtime_ptrs) {
+  if (asan_runtime_ptrs->asan_module == nullptr) {
+    asan_runtime_ptrs->asan_module = GetModuleHandle(L"syzyasan_rtl.dll");
+    if (asan_runtime_ptrs->asan_module != nullptr) {
+      asan_runtime_ptrs->heap_create = reinterpret_cast<HeapCreatePtr>(
+          ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapCreate"));
+      asan_runtime_ptrs->heap_alloc = reinterpret_cast<HeapAllocPtr>(
+          ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapAlloc"));
+      asan_runtime_ptrs->heap_free = reinterpret_cast<HeapFreePtr>(
+          ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapFree"));
+      asan_runtime_ptrs->heap_realloc = reinterpret_cast<HeapReAllocPtr>(
+          ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapReAlloc"));
+      asan_runtime_ptrs->heap_destroy = reinterpret_cast<HeapDestroyPtr>(
+          ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapDestroy"));
+      asan_runtime_ptrs->asan_heap = asan_runtime_ptrs->heap_create(0, 0, 0);
+    }
+  }
+}
+
+// It returns a pointer to a static data structure containing the syzyasan
+// module and the heap methods.
+const AsanRuntimePointers* GetAsanHeapShim() {
+  static AsanRuntimePointers* asan_runtime_ptrs = new AsanRuntimePointers();

[Sigurður Ásgeirsson, 2017/07/31 19:05:52]

better for this to be a straight-up member than a pointer. As-is, the allocation
(may) leak(s) on unload.

[njanevsk, 2017/08/01 15:46:43]

Done.

+  if (asan_runtime_ptrs->asan_module == nullptr) {

[Sigurður Ásgeirsson, 2017/07/31 19:05:52]

This is not threadsafe, plus there's the question of whether you want negative
caching here. I would just shunt this work into the constructor, as there really
isn't any fallback in case of error anyhow.

[njanevsk, 2017/08/01 15:46:43]

Done.

+    ResolveAsanRuntimePointers(asan_runtime_ptrs);
+  }
+  return asan_runtime_ptrs;
+}
+
+inline HANDLE get_heap_handle() {
+  return GetAsanHeapShim()->asan_heap;
+}
+
+}  // namespace
+
+// These symbols override the CRT's implementation of the same functions.
+__declspec(restrict) void* malloc(size_t size) {
+  return GetAsanHeapShim()->heap_alloc(get_heap_handle(), 0, size);
+}
+
+void free(void* ptr) {
+  GetAsanHeapShim()->heap_free(get_heap_handle(), 0, ptr);
+}
+
+__declspec(restrict) void* realloc(void* ptr, size_t size) {
+  return GetAsanHeapShim()->heap_realloc(get_heap_handle(), 0, ptr, size);
+}
+
+__declspec(restrict) void* calloc(size_t n, size_t size) {
+  void* ptr = malloc(size * n);
+  if (ptr != nullptr)
+    ::memset(ptr, 0, size * n);
+  return ptr;
+}
+
+// The symbols
+//   * __acrt_heap
+//   * __acrt_initialize_heap
+//   * __acrt_uninitialize_heap
+//   * _get_heap_handle
+// must be overridden all or none, as they are otherwise supplied
+// by heap_handle.obj in the ucrt.lib file.
+HANDLE __acrt_heap = nullptr;
+
+bool __acrt_initialize_heap() {
+  // TODO(sebmarchand): Check if it is possible to use asan_heap here.
+  __acrt_heap = ::HeapCreate(0, 0, 0);
+  return true;
+}
+
+bool __acrt_uninitialize_heap() {
+  ::HeapDestroy(__acrt_heap);
+  __acrt_heap = nullptr;
+  return true;
+}
+
+intptr_t _get_heap_handle(void) {
+  return reinterpret_cast<intptr_t>(__acrt_heap);
+}
+
+}  // extern "C"