Allocator shims design prototype.

syzygy/integration_tests/allocator_shim.cc

Index: syzygy/integration_tests/allocator_shim.cc
diff --git a/syzygy/integration_tests/allocator_shim.cc b/syzygy/integration_tests/allocator_shim.cc
new file mode 100644
index 0000000000000000000000000000000000000000..e62bd97ef4e34bb732f695ea59391dc486c9f7ca
--- /dev/null
+++ b/syzygy/integration_tests/allocator_shim.cc
@@ -0,0 +1,120 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file defines symbols to override the same functions in the Visual C++
+// CRT implementation.
+
+#include <windows.h>
+#include <malloc.h>
+
+extern "C" {
+
+namespace {
+
+// The following typedefs are function pointers for the functions:
+// heap_create, heap_destroy, heap_alloc, heap_realloc, and heap_free
+// from the ucrt library that we are overwritting with syzyasan_rtl.dll.

[Sébastien Marchand, 2017/07/25 20:07:07]

That's not true, these functions are heap pointers for the heap functions
provided by SyzyAsan, we're not overwriting any of the heap_* functions here,
we're just using them in the memory management functions (malloc/free/...).

So maybe you could just say: "Definition of the heap management functions
exposed by the SyzyAsan runtime library"?

[njanevsk, 2017/07/26 16:23:45]

Done.

+typedef HANDLE(WINAPI* HeapCreatePtr)(DWORD, SIZE_T, SIZE_T);
+typedef BOOL(WINAPI* HeapDestroyPtr)(HANDLE);
+typedef LPVOID(WINAPI* HeapAllocPtr)(HANDLE, DWORD, SIZE_T);
+typedef LPVOID(WINAPI* HeapReAllocPtr)(HANDLE, DWORD, LPVOID, SIZE_T);
+typedef BOOL(WINAPI* HeapFreePtr)(HANDLE, DWORD, LPVOID);
+
+struct AsanRuntimePointers {
+  HANDLE asan_heap = nullptr;
+  HMODULE asan_module = nullptr;
+  HeapCreatePtr heap_create = nullptr;
+  HeapAllocPtr heap_alloc = nullptr;
+  HeapFreePtr heap_free = nullptr;
+  HeapReAllocPtr heap_realloc = nullptr;
+};
+
+AsanRuntimePointers asan_runtime_ptrs;
+
+void resolve_asan_runtime_pointers() {

[Sébastien Marchand, 2017/07/25 20:07:07]

CamelCase.

[njanevsk, 2017/07/26 16:23:45]

Done.

+  if (asan_shim.asan_module == nullptr) {
+    asan_shim.asan_module = GetModuleHandle(L"syzyasan_rtl.dll");
+    asan_shim.heap_create = reinterpret_cast<HeapCreatePtr>(
+        ::GetProcAddress(asan_shim.asan_module, "asan_HeapCreate"));
+    asan_shim.heap_alloc = reinterpret_cast<HeapAllocPtr>(
+        ::GetProcAddress(asan_shim.asan_module, "asan_HeapAlloc"));
+    asan_shim.heap_free = reinterpret_cast<HeapFreePtr>(
+        ::GetProcAddress(asan_shim.asan_module, "asan_HeapFree"));
+    asan_shim.heap_realloc = reinterpret_cast<HeapReAllocPtr>(
+        ::GetProcAddress(asan_shim.asan_module, "asan_HeapReAlloc"));
+  }
+}
+
+inline HANDLE get_heap_handle() {
+  resolve_asan_runtime_pointers();
+  return asan_shim.asan_heap;
+}
+
+}  // namespace
+
+// This function behaves similarly to MSVC's _set_new_mode.
+// If flag is 0 (default), calls to malloc will behave normally.
+// If flag is 1, calls to malloc will behave like calls to new,
+// and the std_new_handler will be invoked on failure.
+// Returns the previous mode.
+//
+// Replaces _set_new_mode in ucrt\heap\new_mode.cpp
+int _set_new_mode(int flag) {
+  return 0;
+}
+
+// Replaces _query_new_mode in ucrt\heap\new_mode.cpp
+int _query_new_mode() {
+  return 0;
+}
+
+// These symbols override the CRT's implementation of the same functions.
+__declspec(restrict) void* malloc(size_t size) {
+  resolve_asan_runtime_pointers();

[Sébastien Marchand, 2017/07/25 20:07:07]

I don't really like having these calls in every functions, what about adding
something like:
const AsanRuntimePointers* GetAsanHeapShim() {
  static AsanRuntimePointers asan_runtime_ptrs;
  if (asan_shim.asan_module == nullptr) {
     resolve_asan_runtime_pointers(&asan_runtimr_ptrs);
  }
  return &asan_runtime_ptrs;
}

and in these functions you'll just do:
GetAsanHeapShim()->heap_free(...);

[njanevsk, 2017/07/26 16:23:45]

Done.

+  return asan_shim.heap_alloc(get_heap_handle(), 0, size);
+}
+
+void free(void* ptr) {
+  resolve_asan_runtime_pointers();
+  asan_shim.heap_free(get_heap_handle(), 0, ptr);
+}
+
+__declspec(restrict) void* realloc(void* ptr, size_t size) {
+  resolve_asan_runtime_pointers();
+  return asan_shim.heap_realloc(get_heap_handle(), 0, ptr, size);
+}
+
+__declspec(restrict) void* calloc(size_t n, size_t size) {
+  resolve_asan_runtime_pointers();
+  void* ptr = malloc(size * n);
+  if (ptr != nullptr)
+    ::memset(ptr, 0, size * n);
+  return ptr;
+}
+
+// The symbols
+//   * __acrt_heap
+//   * __acrt_initialize_heap
+//   * __acrt_uninitialize_heap
+//   * _get_heap_handle
+// must be overridden all or none, as they are otherwise supplied
+// by heap_handle.obj in the ucrt.lib file.
+HANDLE __acrt_heap = nullptr;
+
+bool __acrt_initialize_heap() {
+  __acrt_heap = ::HeapCreate(0, 0, 0);
+  return true;
+}
+
+bool __acrt_uninitialize_heap() {
+  ::HeapDestroy(__acrt_heap);
+  __acrt_heap = nullptr;
+  return true;
+}
+
+intptr_t _get_heap_handle(void) {
+  return reinterpret_cast<intptr_t>(__acrt_heap);
+}
+
+}  // extern "C"