Allocator shims design prototype.

syzygy/integration_tests/allocator_shim.cc

Index: syzygy/integration_tests/allocator_shim.cc
diff --git a/syzygy/integration_tests/allocator_shim.cc b/syzygy/integration_tests/allocator_shim.cc
new file mode 100644
index 0000000000000000000000000000000000000000..b8e0235f5972dfffd913da7bc07cab63d2c111af
--- /dev/null
+++ b/syzygy/integration_tests/allocator_shim.cc
@@ -0,0 +1,107 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file defines symbols to override the same functions in the Visual C++
+// CRT implementation.
+
+#include <windows.h>
+#include <malloc.h>
+
+extern "C" {
+
+namespace {
+
+// Definitions of the following heap management functions provided by
+// the SyzyAsan library:
+// heap_create, heap_destroy, heap_alloc, heap_realloc, and heap_free
+typedef HANDLE(WINAPI* HeapCreatePtr)(DWORD, SIZE_T, SIZE_T);
+typedef BOOL(WINAPI* HeapDestroyPtr)(HANDLE);
+typedef LPVOID(WINAPI* HeapAllocPtr)(HANDLE, DWORD, SIZE_T);
+typedef LPVOID(WINAPI* HeapReAllocPtr)(HANDLE, DWORD, LPVOID, SIZE_T);
+typedef BOOL(WINAPI* HeapFreePtr)(HANDLE, DWORD, LPVOID);
+
+struct AsanRuntimePointers {
+  HANDLE asan_heap = nullptr;
+  HMODULE asan_module = nullptr;
+  HeapCreatePtr heap_create = nullptr;
+  HeapAllocPtr heap_alloc = nullptr;
+  HeapFreePtr heap_free = nullptr;
+  HeapReAllocPtr heap_realloc = nullptr;
+};
+
+// AsanRuntimePointers __attribute__((no_sanitize_address)) asan_runtime_ptrs;
+
+void ResolveAsanRuntimePointers(AsanRuntimePointers* asan_runtime_ptrs) {

[Sébastien Marchand, 2017/07/26 19:12:02]

Add a comment to describe what this function does.

[njanevsk, 2017/07/27 18:02:00]

Done.

+  if (asan_runtime_ptrs->asan_module == nullptr) {
+    asan_runtime_ptrs->asan_module = GetModuleHandle(L"syzyasan_rtl.dll");
+    asan_runtime_ptrs->heap_create = reinterpret_cast<HeapCreatePtr>(
+        ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapCreate"));
+    asan_runtime_ptrs->heap_alloc = reinterpret_cast<HeapAllocPtr>(
+        ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapAlloc"));
+    asan_runtime_ptrs->heap_free = reinterpret_cast<HeapFreePtr>(
+        ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapFree"));
+    asan_runtime_ptrs->heap_realloc = reinterpret_cast<HeapReAllocPtr>(
+        ::GetProcAddress(asan_runtime_ptrs->asan_module, "asan_HeapReAlloc"));
+  }
+}
+
+const AsanRuntimePointers* GetAsanHeapShim() {

[Sébastien Marchand, 2017/07/26 19:12:02]

Ditto, comment.

[njanevsk, 2017/07/27 18:01:59]

Done.

+  static AsanRuntimePointers asan_runtime_ptrs;

[Sébastien Marchand, 2017/07/26 19:12:02]

Add "= {}"

[njanevsk, 2017/07/27 18:01:59]

Done.

+  if (asan_runtime_ptrs.asan_module == nullptr) {
+    ResolveAsanRuntimePointers(&asan_runtime_ptrs);
+  }
+  return &asan_runtime_ptrs;
+}
+
+inline HANDLE get_heap_handle() {
+  return GetAsanHeapShim()->asan_heap;
+}
+
+}  // namespace
+
+// These symbols override the CRT's implementation of the same functions.
+__declspec(restrict) void* malloc(size_t size) {
+  return GetAsanHeapShim()->heap_alloc(get_heap_handle(), 0, size);
+}
+
+void free(void* ptr) {
+  GetAsanHeapShim()->heap_free(get_heap_handle(), 0, ptr);
+}
+
+__declspec(restrict) void* realloc(void* ptr, size_t size) {
+  return GetAsanHeapShim()->heap_realloc(get_heap_handle(), 0, ptr, size);
+}
+
+__declspec(restrict) void* calloc(size_t n, size_t size) {
+  void* ptr = malloc(size * n);
+  if (ptr != nullptr)
+    ::memset(ptr, 0, size * n);
+  return ptr;
+}
+
+// The symbols
+//   * __acrt_heap
+//   * __acrt_initialize_heap
+//   * __acrt_uninitialize_heap
+//   * _get_heap_handle
+// must be overridden all or none, as they are otherwise supplied
+// by heap_handle.obj in the ucrt.lib file.
+HANDLE __acrt_heap = nullptr;
+
+bool __acrt_initialize_heap() {
+  __acrt_heap = ::HeapCreate(0, 0, 0);

[Sébastien Marchand, 2017/07/26 19:12:02]

Ideally I'd like to use an Asan heap here too but I don't think that it's
possible (was crashing when I tried it locally, I'll need to investigate).

Can you add a TODO (assigned to me) to check if it'll be possible to use an Asan
heap?

[njanevsk, 2017/07/27 18:02:00]

Done.

+  return true;
+}
+
+bool __acrt_uninitialize_heap() {
+  ::HeapDestroy(__acrt_heap);
+  __acrt_heap = nullptr;
+  return true;
+}
+
+intptr_t _get_heap_handle(void) {
+  return reinterpret_cast<intptr_t>(__acrt_heap);
+}
+
+}  // extern "C"