third_party/tcmalloc/chromium/src/tcmalloc.cc - Issue 9320005: [NOT TO COMMIT!] Replace third_party/tcmalloc/chromium with tcmalloc r136 (the latest).

Unified Diff: third_party/tcmalloc/chromium/src/tcmalloc.cc

Issue 9320005: [NOT TO COMMIT!] Replace third_party/tcmalloc/chromium with tcmalloc r136 (the latest). (Closed) Base URL: http://git.chromium.org/git/chromium.git@trunk

Patch Set: Created 8 years, 11 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

« no previous file with comments | « third_party/tcmalloc/chromium/src/system-alloc.cc ('k') | third_party/tcmalloc/chromium/src/tests/debugallocation_test.cc » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')

Index: third_party/tcmalloc/chromium/src/tcmalloc.cc

diff --git a/third_party/tcmalloc/chromium/src/tcmalloc.cc b/third_party/tcmalloc/chromium/src/tcmalloc.cc

index 51fe5b31ecd17639e6e58b0d1ec60a66c01cb333..425e90b9b240eec8c42c1677d228d67ff7e10931 100644

--- a/third_party/tcmalloc/chromium/src/tcmalloc.cc

+++ b/third_party/tcmalloc/chromium/src/tcmalloc.cc

@@ -93,9 +93,6 @@

#ifdef HAVE_SYS_CDEFS_H

#include <sys/cdefs.h> // for __THROW

#endif

-#ifdef HAVE_FEATURES_H

-#include <features.h> // for __GLIBC__

-#endif

#if defined HAVE_STDINT_H

#include <stdint.h>

#elif defined HAVE_INTTYPES_H

@@ -122,8 +119,8 @@

#include "base/spinlock.h" // for SpinLockHolder

#include "central_freelist.h" // for CentralFreeListPadded

#include "common.h" // for StackTrace, kPageShift, etc

-#include "free_list.h" // for FL_Init

#include "internal_logging.h" // for ASSERT, TCMalloc_Printer, etc

+#include "linked_list.h" // for SLL_SetNext

#include "malloc_hook-inl.h" // for MallocHook::InvokeNewHook, etc

#include "page_heap.h" // for PageHeap, PageHeap::Stats

#include "page_heap_allocator.h" // for PageHeapAllocator

@@ -150,17 +147,28 @@

# define WIN32_DO_PATCHING 1

#endif

-// GLibc 2.14+ requires the hook functions be declared volatile, based on the

-// value of the define __MALLOC_HOOK_VOLATILE. For compatibility with

-// older/non-GLibc implementations, provide an empty definition.

-#if !defined(__MALLOC_HOOK_VOLATILE)

-#define __MALLOC_HOOK_VOLATILE

-#endif

+// Some windows file somewhere (at least on cygwin) #define's small (!)

+#undef small

using STL_NAMESPACE::max;

using STL_NAMESPACE::numeric_limits;

using STL_NAMESPACE::vector;

+#include "libc_override.h"

+// __THROW is defined in glibc (via <sys/cdefs.h>). It means,

+// counter-intuitively, "This function will never throw an exception."

+// It's an optional optimization tool, but we may need to use it to

+// match glibc prototypes.

+#ifndef __THROW // I guess we're not on a glibc system

+# define __THROW // __THROW is just an optimization, so ok to make it ""

+#endif

using tcmalloc::AlignmentForSize;

+using tcmalloc::kLog;

+using tcmalloc::kCrash;

+using tcmalloc::kCrashWithStats;

+using tcmalloc::Log;

using tcmalloc::PageHeap;

using tcmalloc::PageHeapAllocator;

using tcmalloc::SizeMap;

@@ -169,21 +177,6 @@ using tcmalloc::StackTrace;

using tcmalloc::Static;

using tcmalloc::ThreadCache;

-// __THROW is defined in glibc systems. It means, counter-intuitively,

-// "This function will never throw an exception." It's an optional

-// optimization tool, but we may need to use it to match glibc prototypes.

-#ifndef __THROW // I guess we're not on a glibc system

-# define __THROW // __THROW is just an optimization, so ok to make it ""

-#endif

-// ---- Double free debug declarations

-static size_t ExcludeSpaceForMark(size_t size);

-static void AddRoomForMark(size_t* size);

-static void ExcludeMarkFromSize(size_t* new_size);

-static void MarkAllocatedRegion(void* ptr);

-static void ValidateAllocatedRegion(void* ptr, size_t cl);

-// ---- End Double free debug declarations

DECLARE_int64(tcmalloc_sample_parameter);

DECLARE_double(tcmalloc_release_rate);

@@ -218,6 +211,7 @@ DEFINE_int64(tcmalloc_large_alloc_report_threshold,

// put all callers of MallocHook::Invoke* in this module into

// ATTRIBUTE_SECTION(google_malloc) section, so that

// MallocHook::GetCallerStackTrace can function accurately.

+#ifndef _WIN32 // windows doesn't have attribute_section, so don't bother

extern "C" {

void* tc_malloc(size_t size) __THROW

ATTRIBUTE_SECTION(google_malloc);

@@ -279,161 +273,7 @@ extern "C" {

size_t tc_malloc_size(void* p) __THROW

ATTRIBUTE_SECTION(google_malloc);

} // extern "C"

-// Override the libc functions to prefer our own instead. This comes

-// first so code in tcmalloc.cc can use the overridden versions. One

-// exception: in windows, by default, we patch our code into these

-// functions (via src/windows/patch_function.cc) rather than override

-// them. In that case, we don't want to do this overriding here.

-#if !defined(WIN32_DO_PATCHING)

-// TODO(mbelshe): Turn off TCMalloc's symbols for libc. We do that

-// elsewhere.

-#ifndef _WIN32

-#if defined(__GNUC__) && !defined(__MACH__)

- // Potentially faster variants that use the gcc alias extension.

- // FreeBSD does support aliases, but apparently not correctly. :-(

- // NOTE: we make many of these symbols weak, but do so in the makefile

- // (via objcopy -W) and not here. That ends up being more portable.

-# define ALIAS(x) __attribute__ ((alias (x)))

-void* operator new(size_t size) throw (std::bad_alloc) ALIAS("tc_new");

-void operator delete(void* p) __THROW ALIAS("tc_delete");

-void* operator new[](size_t size) throw (std::bad_alloc) ALIAS("tc_newarray");

-void operator delete[](void* p) __THROW ALIAS("tc_deletearray");

-void* operator new(size_t size, const std::nothrow_t&) __THROW

- ALIAS("tc_new_nothrow");

-void* operator new[](size_t size, const std::nothrow_t&) __THROW

- ALIAS("tc_newarray_nothrow");

-void operator delete(void* size, const std::nothrow_t&) __THROW

- ALIAS("tc_delete_nothrow");

-void operator delete[](void* size, const std::nothrow_t&) __THROW

- ALIAS("tc_deletearray_nothrow");

-extern "C" {

- void* malloc(size_t size) __THROW ALIAS("tc_malloc");

- void free(void* ptr) __THROW ALIAS("tc_free");

- void* realloc(void* ptr, size_t size) __THROW ALIAS("tc_realloc");

- void* calloc(size_t n, size_t size) __THROW ALIAS("tc_calloc");

- void cfree(void* ptr) __THROW ALIAS("tc_cfree");

- void* memalign(size_t align, size_t s) __THROW ALIAS("tc_memalign");

- void* valloc(size_t size) __THROW ALIAS("tc_valloc");

- void* pvalloc(size_t size) __THROW ALIAS("tc_pvalloc");

- int posix_memalign(void** r, size_t a, size_t s) __THROW

- ALIAS("tc_posix_memalign");

- void malloc_stats(void) __THROW ALIAS("tc_malloc_stats");

- int mallopt(int cmd, int value) __THROW ALIAS("tc_mallopt");

-#ifdef HAVE_STRUCT_MALLINFO

- struct mallinfo mallinfo(void) __THROW ALIAS("tc_mallinfo");

-#endif

- size_t malloc_size(void* p) __THROW ALIAS("tc_malloc_size");

- size_t malloc_usable_size(void* p) __THROW ALIAS("tc_malloc_size");

-} // extern "C"

-#else // #if defined(__GNUC__) && !defined(__MACH__)

-// Portable wrappers

-void* operator new(size_t size) { return tc_new(size); }

-void operator delete(void* p) __THROW { tc_delete(p); }

-void* operator new[](size_t size) { return tc_newarray(size); }

-void operator delete[](void* p) __THROW { tc_deletearray(p); }

-void* operator new(size_t size, const std::nothrow_t& nt) __THROW {

- return tc_new_nothrow(size, nt);

-void* operator new[](size_t size, const std::nothrow_t& nt) __THROW {

- return tc_newarray_nothrow(size, nt);

-void operator delete(void* ptr, const std::nothrow_t& nt) __THROW {

- return tc_delete_nothrow(ptr, nt);

-void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW {

- return tc_deletearray_nothrow(ptr, nt);

-extern "C" {

- void* malloc(size_t s) __THROW { return tc_malloc(s); }

- void free(void* p) __THROW { tc_free(p); }

- void* realloc(void* p, size_t s) __THROW { return tc_realloc(p, s); }

- void* calloc(size_t n, size_t s) __THROW { return tc_calloc(n, s); }

- void cfree(void* p) __THROW { tc_cfree(p); }

- void* memalign(size_t a, size_t s) __THROW { return tc_memalign(a, s); }

- void* valloc(size_t s) __THROW { return tc_valloc(s); }

- void* pvalloc(size_t s) __THROW { return tc_pvalloc(s); }

- int posix_memalign(void** r, size_t a, size_t s) __THROW {

- return tc_posix_memalign(r, a, s);

- }

- void malloc_stats(void) __THROW { tc_malloc_stats(); }

- int mallopt(int cmd, int v) __THROW { return tc_mallopt(cmd, v); }

-#ifdef HAVE_STRUCT_MALLINFO

- struct mallinfo mallinfo(void) __THROW { return tc_mallinfo(); }

-#endif

- size_t malloc_size(void* p) __THROW { return tc_malloc_size(p); }

- size_t malloc_usable_size(void* p) __THROW { return tc_malloc_size(p); }

-} // extern "C"

-#endif // #if defined(__GNUC__)

-// Some library routines on RedHat 9 allocate memory using malloc()

-// and free it using __libc_free() (or vice-versa). Since we provide

-// our own implementations of malloc/free, we need to make sure that

-// the __libc_XXX variants (defined as part of glibc) also point to

-// the same implementations.

-#ifdef __GLIBC__ // only glibc defines __libc_*

-extern "C" {

-#ifdef ALIAS

- void* __libc_malloc(size_t size) ALIAS("tc_malloc");

- void __libc_free(void* ptr) ALIAS("tc_free");

- void* __libc_realloc(void* ptr, size_t size) ALIAS("tc_realloc");

- void* __libc_calloc(size_t n, size_t size) ALIAS("tc_calloc");

- void __libc_cfree(void* ptr) ALIAS("tc_cfree");

- void* __libc_memalign(size_t align, size_t s) ALIAS("tc_memalign");

- void* __libc_valloc(size_t size) ALIAS("tc_valloc");

- void* __libc_pvalloc(size_t size) ALIAS("tc_pvalloc");

- int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign");

-#else // #ifdef ALIAS

- void* __libc_malloc(size_t size) { return malloc(size); }

- void __libc_free(void* ptr) { free(ptr); }

- void* __libc_realloc(void* ptr, size_t size) { return realloc(ptr, size); }

- void* __libc_calloc(size_t n, size_t size) { return calloc(n, size); }

- void __libc_cfree(void* ptr) { cfree(ptr); }

- void* __libc_memalign(size_t align, size_t s) { return memalign(align, s); }

- void* __libc_valloc(size_t size) { return valloc(size); }

- void* __libc_pvalloc(size_t size) { return pvalloc(size); }

- int __posix_memalign(void** r, size_t a, size_t s) {

- return posix_memalign(r, a, s);

- }

-#endif // #ifdef ALIAS

-} // extern "C"

-#endif // ifdef __GLIBC__

-#if defined(__GLIBC__) && defined(HAVE_MALLOC_H)

-// If we're using glibc, then override glibc malloc hooks to make sure that even

-// if calls fall through to ptmalloc (due to dlopen() with RTLD_DEEPBIND or what

-// not), ptmalloc will use TCMalloc.

-static void* tc_ptmalloc_malloc_hook(size_t size, const void* caller) {

- return tc_malloc(size);

-void* (*__MALLOC_HOOK_VOLATILE __malloc_hook)(

- size_t size, const void* caller) = tc_ptmalloc_malloc_hook;

-static void* tc_ptmalloc_realloc_hook(

- void* ptr, size_t size, const void* caller) {

- return tc_realloc(ptr, size);

-void* (*__MALLOC_HOOK_VOLATILE __realloc_hook)(

- void* ptr, size_t size, const void* caller) = tc_ptmalloc_realloc_hook;

-static void tc_ptmalloc_free_hook(void* ptr, const void* caller) {

- tc_free(ptr);

-void (*__MALLOC_HOOK_VOLATILE __free_hook)(void* ptr, const void* caller) = tc_ptmalloc_free_hook;

-#endif

#endif // #ifndef _WIN32

-#undef ALIAS

-#endif // #ifndef(WIN32_DO_PATCHING)

// ----------------------- IMPLEMENTATION -------------------------------

@@ -446,16 +286,18 @@ static int tc_new_mode = 0; // See tc_set_new_mode().

// required) kind of exception handling for these routines.

namespace {

void InvalidFree(void* ptr) {

- CRASH("Attempt to free invalid pointer: %p\n", ptr);

+ Log(kCrash, __FILE__, __LINE__, "Attempt to free invalid pointer", ptr);

}

-size_t InvalidGetSizeForRealloc(void* old_ptr) {

- CRASH("Attempt to realloc invalid pointer: %p\n", old_ptr);

+size_t InvalidGetSizeForRealloc(const void* old_ptr) {

+ Log(kCrash, __FILE__, __LINE__,

+ "Attempt to realloc invalid pointer", old_ptr);

return 0;

}

-size_t InvalidGetAllocatedSize(void* ptr) {

- CRASH("Attempt to get the size of an invalid pointer: %p\n", ptr);

+size_t InvalidGetAllocatedSize(const void* ptr) {

+ Log(kCrash, __FILE__, __LINE__,

+ "Attempt to get the size of an invalid pointer", ptr);

return 0;

}

} // unnamed namespace

@@ -470,15 +312,18 @@ struct TCMallocStats {

};

// Get stats into "r". Also get per-size-class counts if class_count != NULL

-static void ExtractStats(TCMallocStats* r, uint64_t* class_count) {

+static void ExtractStats(TCMallocStats* r, uint64_t* class_count,

+ PageHeap::SmallSpanStats* small_spans,

+ PageHeap::LargeSpanStats* large_spans) {

r->central_bytes = 0;

r->transfer_bytes = 0;

for (int cl = 0; cl < kNumClasses; ++cl) {

const int length = Static::central_cache()[cl].length();

const int tc_length = Static::central_cache()[cl].tc_length();

+ const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();

const size_t size = static_cast<uint64_t>(

Static::sizemap()->ByteSizeForClass(cl));

- r->central_bytes += (size * length);

+ r->central_bytes += (size * length) + cache_overhead;

r->transfer_bytes += (size * tc_length);

if (class_count) class_count[cl] = length + tc_length;

}

@@ -490,14 +335,30 @@ static void ExtractStats(TCMallocStats* r, uint64_t* class_count) {

ThreadCache::GetThreadStats(&r->thread_bytes, class_count);

r->metadata_bytes = tcmalloc::metadata_system_bytes();

r->pageheap = Static::pageheap()->stats();

+ if (small_spans != NULL) {

+ Static::pageheap()->GetSmallSpanStats(small_spans);

+ }

+ if (large_spans != NULL) {

+ Static::pageheap()->GetLargeSpanStats(large_spans);

+ }

}

+static double PagesToMiB(uint64_t pages) {

+ return (pages << kPageShift) / 1048576.0;

// WRITE stats to "out"

static void DumpStats(TCMalloc_Printer* out, int level) {

TCMallocStats stats;

uint64_t class_count[kNumClasses];

- ExtractStats(&stats, (level >= 2 ? class_count : NULL));

+ PageHeap::SmallSpanStats small;

+ PageHeap::LargeSpanStats large;

+ if (level >= 2) {

+ ExtractStats(&stats, class_count, &small, &large);

+ } else {

+ ExtractStats(&stats, NULL, NULL, NULL);

+ }

static const double MiB = 1048576.0;

@@ -512,15 +373,6 @@ static void DumpStats(TCMalloc_Printer* out, int level) {

- stats.transfer_bytes

- stats.thread_bytes);

- out->printf(

- "WASTE: %7.1f MiB committed but not used\n"

- "WASTE: %7.1f MiB bytes committed, %7.1f MiB bytes in use\n"

- "WASTE: committed/used ratio of %f\n",

- (stats.pageheap.committed_bytes - bytes_in_use_by_app) / MiB,

- stats.pageheap.committed_bytes / MiB,

- bytes_in_use_by_app / MiB,

- stats.pageheap.committed_bytes / static_cast<double>(bytes_in_use_by_app)

- );

#ifdef TCMALLOC_SMALL_BUT_SLOW

out->printf(

"NOTE: SMALL MEMORY MODEL IS IN USE, PERFORMANCE MAY SUFFER.\n");

@@ -528,7 +380,6 @@ static void DumpStats(TCMalloc_Printer* out, int level) {

out->printf(

"------------------------------------------------\n"

"MALLOC: %12" PRIu64 " (%7.1f MiB) Bytes in use by application\n"

- "MALLOC: %12" PRIu64 " (%7.1f MB) Bytes committed\n"

"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in page heap freelist\n"

"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in central cache freelist\n"

"MALLOC: + %12" PRIu64 " (%7.1f MiB) Bytes in transfer cache freelist\n"

@@ -549,7 +400,6 @@ static void DumpStats(TCMalloc_Printer* out, int level) {

"Bytes released to the OS take up virtual address space"

" but no physical memory.\n",

bytes_in_use_by_app, bytes_in_use_by_app / MiB,

- stats.pageheap.committed_bytes, stats.pageheap.committed_bytes / MiB,

stats.pageheap.free_bytes, stats.pageheap.free_bytes / MiB,

stats.central_bytes, stats.central_bytes / MiB,

stats.transfer_bytes, stats.transfer_bytes / MiB,

@@ -581,8 +431,48 @@ static void DumpStats(TCMalloc_Printer* out, int level) {

}

- SpinLockHolder h(Static::pageheap_lock());

- Static::pageheap()->Dump(out);

+ // append page heap info

+ int nonempty_sizes = 0;

+ for (int s = 0; s < kMaxPages; s++) {

+ if (small.normal_length[s] + small.returned_length[s] > 0) {

+ nonempty_sizes++;

+ }

+ out->printf("------------------------------------------------\n");

+ out->printf("PageHeap: %d sizes; %6.1f MiB free; %6.1f MiB unmapped\n",

+ nonempty_sizes, stats.pageheap.free_bytes / MiB,

+ stats.pageheap.unmapped_bytes / MiB);

+ out->printf("------------------------------------------------\n");

+ uint64_t total_normal = 0;

+ uint64_t total_returned = 0;

+ for (int s = 0; s < kMaxPages; s++) {

+ const int n_length = small.normal_length[s];

+ const int r_length = small.returned_length[s];

+ if (n_length + r_length > 0) {

+ uint64_t n_pages = s * n_length;

+ uint64_t r_pages = s * r_length;

+ total_normal += n_pages;

+ total_returned += r_pages;

+ out->printf("%6u pages * %6u spans ~ %6.1f MiB; %6.1f MiB cum"

+ "; unmapped: %6.1f MiB; %6.1f MiB cum\n",

+ s,

+ (n_length + r_length),

+ PagesToMiB(n_pages + r_pages),

+ PagesToMiB(total_normal + total_returned),

+ PagesToMiB(r_pages),

+ PagesToMiB(total_returned));

+ }

+ total_normal += large.normal_pages;

+ total_returned += large.returned_pages;

+ out->printf(">255 large * %6u spans ~ %6.1f MiB; %6.1f MiB cum"

+ "; unmapped: %6.1f MiB; %6.1f MiB cum\n",

+ static_cast<unsigned int>(large.spans),

+ PagesToMiB(large.normal_pages + large.returned_pages),

+ PagesToMiB(total_normal + total_returned),

+ PagesToMiB(large.returned_pages),

+ PagesToMiB(total_returned));

}

@@ -612,8 +502,9 @@ static void** DumpHeapGrowthStackTraces() {

void** result = new void*[needed_slots];

if (result == NULL) {

- MESSAGE("tcmalloc: allocation failed for stack trace slots",

- needed_slots * sizeof(*result));

+ Log(kLog, __FILE__, __LINE__,

+ "tcmalloc: allocation failed for stack trace slots",

+ needed_slots * sizeof(*result));

return NULL;

}

@@ -739,7 +630,7 @@ class TCMallocImplementation : public MallocExtension {

if (strcmp(name, "generic.current_allocated_bytes") == 0) {

TCMallocStats stats;

- ExtractStats(&stats, NULL);

+ ExtractStats(&stats, NULL, NULL, NULL);

*value = stats.pageheap.system_bytes

- stats.thread_bytes

- stats.central_bytes

@@ -751,7 +642,7 @@ class TCMallocImplementation : public MallocExtension {

if (strcmp(name, "generic.heap_size") == 0) {

TCMallocStats stats;

- ExtractStats(&stats, NULL);

+ ExtractStats(&stats, NULL, NULL, NULL);

*value = stats.pageheap.system_bytes;

return true;

}

@@ -785,7 +676,7 @@ class TCMallocImplementation : public MallocExtension {

if (strcmp(name, "tcmalloc.current_total_thread_cache_bytes") == 0) {

TCMallocStats stats;

- ExtractStats(&stats, NULL);

+ ExtractStats(&stats, NULL, NULL, NULL);

*value = stats.thread_bytes;

return true;

}

@@ -866,7 +757,26 @@ class TCMallocImplementation : public MallocExtension {

// This just calls GetSizeWithCallback, but because that's in an

// unnamed namespace, we need to move the definition below it in the

// file.

- virtual size_t GetAllocatedSize(void* ptr);

+ virtual size_t GetAllocatedSize(const void* ptr);

+ // This duplicates some of the logic in GetSizeWithCallback, but is

+ // faster. This is important on OS X, where this function is called

+ // on every allocation operation.

+ virtual Ownership GetOwnership(const void* ptr) {

+ const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;

+ // The rest of tcmalloc assumes that all allocated pointers use at

+ // most kAddressBits bits. If ptr doesn't, then it definitely

+ // wasn't alloacted by tcmalloc.

+ if ((p >> (kAddressBits - kPageShift)) > 0) {

+ return kNotOwned;

+ }

+ size_t cl = Static::pageheap()->GetSizeClassIfCached(p);

+ if (cl != 0) {

+ return kOwned;

+ }

+ const Span *span = Static::pageheap()->GetDescriptor(p);

+ return span ? kOwned : kNotOwned;

+ }

virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {

static const char* kCentralCacheType = "tcmalloc.central";

@@ -921,42 +831,39 @@ class TCMallocImplementation : public MallocExtension {

}

// append page heap info

- int64 page_count_normal[kMaxPages];

- int64 page_count_returned[kMaxPages];

- int64 span_count_normal;

- int64 span_count_returned;

+ PageHeap::SmallSpanStats small;

+ PageHeap::LargeSpanStats large;

{

SpinLockHolder h(Static::pageheap_lock());

- Static::pageheap()->GetClassSizes(page_count_normal,

- page_count_returned,

- &span_count_normal,

- &span_count_returned);

+ Static::pageheap()->GetSmallSpanStats(&small);

+ Static::pageheap()->GetLargeSpanStats(&large);

}

- // spans: mapped

+ // large spans: mapped

MallocExtension::FreeListInfo span_info;

span_info.type = kLargeSpanType;

span_info.max_object_size = (numeric_limits<size_t>::max)();

span_info.min_object_size = kMaxPages << kPageShift;

- span_info.total_bytes_free = span_count_normal << kPageShift;

+ span_info.total_bytes_free = large.normal_pages << kPageShift;

v->push_back(span_info);

- // spans: unmapped

+ // large spans: unmapped

span_info.type = kLargeUnmappedSpanType;

- span_info.total_bytes_free = span_count_returned << kPageShift;

+ span_info.total_bytes_free = large.returned_pages << kPageShift;

v->push_back(span_info);

+ // small spans

for (int s = 1; s < kMaxPages; s++) {

MallocExtension::FreeListInfo i;

i.max_object_size = (s << kPageShift);

i.min_object_size = ((s - 1) << kPageShift);

i.type = kPageHeapType;

- i.total_bytes_free = (s << kPageShift) * page_count_normal[s];

+ i.total_bytes_free = (s << kPageShift) * small.normal_length[s];

v->push_back(i);

i.type = kPageHeapUnmappedType;

- i.total_bytes_free = (s << kPageShift) * page_count_returned[s];

+ i.total_bytes_free = (s << kPageShift) * small.returned_length[s];

v->push_back(i);

}

@@ -981,10 +888,7 @@ TCMallocGuard::TCMallocGuard() {

// Check whether the kernel also supports TLS (needs to happen at runtime)

tcmalloc::CheckIfKernelSupportsTLS();

#endif

-#ifdef WIN32_DO_PATCHING

- // patch the windows VirtualAlloc, etc.

- PatchWindowsFunctions(); // defined in windows/patch_functions.cc

-#endif

+ ReplaceSystemAlloc(); // defined in libc_override_*.h

tc_free(tc_malloc(1));

ThreadCache::InitTSD();

tc_free(tc_malloc(1));

@@ -1029,7 +933,6 @@ static inline bool CheckCachedSizeClass(void *ptr) {

static inline void* CheckedMallocResult(void *result) {

ASSERT(result == NULL || CheckCachedSizeClass(result));

- MarkAllocatedRegion(result);

return result;

}

@@ -1081,8 +984,8 @@ static void ReportLargeAlloc(Length num_pages, void* result) {

static const int N = 1000;

char buffer[N];

TCMalloc_Printer printer(buffer, N);

- printer.printf("tcmalloc: large alloc %llu bytes == %p @ ",

- static_cast<unsigned long long>(num_pages) << kPageShift,

+ printer.printf("tcmalloc: large alloc %"PRIu64" bytes == %p @ ",

+ static_cast<uint64>(num_pages) << kPageShift,

result);

for (int i = 0; i < stack.depth; i++) {

printer.printf(" %p", stack.stack[i]);

@@ -1094,7 +997,7 @@ static void ReportLargeAlloc(Length num_pages, void* result) {

inline void* cpp_alloc(size_t size, bool nothrow);

inline void* do_malloc(size_t size);

-// TODO(willchan): Investigate whether or not inlining this much is harmful to

+// TODO(willchan): Investigate whether or not lining this much is harmful to

// performance.

// This is equivalent to do_malloc() except when tc_new_mode is set to true.

// Otherwise, it will run the std::new_handler if set.

@@ -1149,8 +1052,6 @@ inline void* do_malloc_pages(ThreadCache* heap, size_t size) {

}

inline void* do_malloc(size_t size) {

- AddRoomForMark(&size);

void* ret = NULL;

// The following call forces module initialization

@@ -1161,15 +1062,13 @@ inline void* do_malloc(size_t size) {

if ((FLAGS_tcmalloc_sample_parameter > 0) && heap->SampleAllocation(size)) {

ret = DoSampledAllocation(size);

- MarkAllocatedRegion(ret);

} else {

- // The common case, and also the simplest. This just pops the

+ // The common case, and also the simplest. This just pops the

// size-appropriate freelist, after replenishing it if it's empty.

ret = CheckedMallocResult(heap->Allocate(size, cl));

}

} else {

ret = do_malloc_pages(heap, size);

- MarkAllocatedRegion(ret);

}

if (ret == NULL) errno = ENOMEM;

return ret;

@@ -1196,7 +1095,15 @@ static inline ThreadCache* GetCacheIfPresent() {

// It is used primarily by windows code which wants a specialized callback.

inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) {

if (ptr == NULL) return;

- ASSERT(Static::pageheap() != NULL); // Should not call free() before malloc()

+ if (Static::pageheap() == NULL) {

+ // We called free() before malloc(). This can occur if the

+ // (system) malloc() is called before tcmalloc is loaded, and then

+ // free() is called after tcmalloc is loaded (and tc_free has

+ // replaced free), but before the global constructor has run that

+ // sets up the tcmalloc data structures.

+ (*invalid_free_fn)(ptr); // Decide how to handle the bad free request

+ return;

+ }

const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;

Span* span = NULL;

size_t cl = Static::pageheap()->GetSizeClassIfCached(p);

@@ -1217,9 +1124,6 @@ inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) {

cl = span->sizeclass;

Static::pageheap()->CacheSizeClass(p, cl);

}

- ValidateAllocatedRegion(ptr, cl);

if (cl != 0) {

ASSERT(!Static::pageheap()->GetDescriptor(p)->sample);

ThreadCache* heap = GetCacheIfPresent();

@@ -1227,7 +1131,7 @@ inline void do_free_with_callback(void* ptr, void (*invalid_free_fn)(void*)) {

heap->Deallocate(ptr, cl);

} else {

// Delete directly into central cache

- tcmalloc::FL_Init(ptr);

+ tcmalloc::SLL_SetNext(ptr, NULL);

Static::central_cache()[cl].InsertRange(ptr, ptr, 1);

}

} else {

@@ -1249,8 +1153,10 @@ inline void do_free(void* ptr) {

return do_free_with_callback(ptr, &InvalidFree);

}

-inline size_t GetSizeWithCallback(void* ptr,

- size_t (*invalid_getsize_fn)(void*)) {

+// NOTE: some logic here is duplicated in GetOwnership (above), for

+// speed. If you change this function, look at that one too.

+inline size_t GetSizeWithCallback(const void* ptr,

+ size_t (*invalid_getsize_fn)(const void*)) {

if (ptr == NULL)

return 0;

const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;

@@ -1258,7 +1164,7 @@ inline size_t GetSizeWithCallback(void* ptr,

if (cl != 0) {

return Static::sizemap()->ByteSizeForClass(cl);

} else {

- Span *span = Static::pageheap()->GetDescriptor(p);

+ const Span *span = Static::pageheap()->GetDescriptor(p);

if (span == NULL) { // means we do not own this memory

return (*invalid_getsize_fn)(ptr);

} else if (span->sizeclass != 0) {

@@ -1275,8 +1181,7 @@ inline size_t GetSizeWithCallback(void* ptr,

inline void* do_realloc_with_callback(

void* old_ptr, size_t new_size,

void (*invalid_free_fn)(void*),

- size_t (*invalid_get_size_fn)(void*)) {

- AddRoomForMark(&new_size);

+ size_t (*invalid_get_size_fn)(const void*)) {

// Get the size of the old entry

const size_t old_size = GetSizeWithCallback(old_ptr, invalid_get_size_fn);

@@ -1295,7 +1200,6 @@ inline void* do_realloc_with_callback(

if (new_size > old_size && new_size < lower_bound_to_grow) {

new_ptr = do_malloc_or_cpp_alloc(lower_bound_to_grow);

}

- ExcludeMarkFromSize(&new_size); // do_malloc will add space if needed.

if (new_ptr == NULL) {

// Either new_size is not a tiny increment, or last do_malloc failed.

new_ptr = do_malloc_or_cpp_alloc(new_size);

@@ -1314,7 +1218,6 @@ inline void* do_realloc_with_callback(

} else {

// We still need to call hooks to report the updated size:

MallocHook::InvokeDeleteHook(old_ptr);

- ExcludeMarkFromSize(&new_size);

MallocHook::InvokeNewHook(old_ptr, new_size);

return old_ptr;

}

@@ -1335,8 +1238,6 @@ inline void* do_realloc(void* old_ptr, size_t new_size) {

void* do_memalign(size_t align, size_t size) {

ASSERT((align & (align - 1)) == 0);

ASSERT(align > 0);

- // Marked in CheckMallocResult(), which is also inside SpanToMallocResult().

- AddRoomForMark(&size);

if (size + align < size) return NULL; // Overflow

// Fall back to malloc if we would already align this memory access properly.

@@ -1421,7 +1322,7 @@ inline int do_mallopt(int cmd, int value) {

#ifdef HAVE_STRUCT_MALLINFO

inline struct mallinfo do_mallinfo() {

TCMallocStats stats;

- ExtractStats(&stats, NULL);

+ ExtractStats(&stats, NULL, NULL, NULL);

// Just some of the fields are filled in.

struct mallinfo info;

@@ -1545,9 +1446,10 @@ void* cpp_memalign(size_t align, size_t size) {

} // end unnamed namespace

// As promised, the definition of this function, declared above.

-size_t TCMallocImplementation::GetAllocatedSize(void* ptr) {

- return ExcludeSpaceForMark(

- GetSizeWithCallback(ptr, &InvalidGetAllocatedSize));

+size_t TCMallocImplementation::GetAllocatedSize(const void* ptr) {

+ ASSERT(TCMallocImplementation::GetOwnership(ptr)

+ != TCMallocImplementation::kNotOwned);

+ return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);

}

void TCMallocImplementation::MarkThreadBusy() {

@@ -1744,194 +1646,7 @@ extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {

#endif

extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {

- return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);

+ return MallocExtension::instance()->GetAllocatedSize(ptr);

}

-// Override __libc_memalign in libc on linux boxes specially.

-// They have a bug in libc that causes them to (very rarely) allocate

-// with __libc_memalign() yet deallocate with free() and the

-// definitions above don't catch it.

-// This function is an exception to the rule of calling MallocHook method

-// from the stack frame of the allocation function;

-// heap-checker handles this special case explicitly.

-static void *MemalignOverride(size_t align, size_t size, const void *caller)

- __THROW ATTRIBUTE_SECTION(google_malloc);

-static void *MemalignOverride(size_t align, size_t size, const void *caller)

- __THROW {

- void* result = do_memalign_or_cpp_memalign(align, size);

- MallocHook::InvokeNewHook(result, size);

- return result;

-void *(*__MALLOC_HOOK_VOLATILE __memalign_hook)(size_t, size_t, const void *) = MemalignOverride;

#endif // TCMALLOC_USING_DEBUGALLOCATION

-// ---Double free() debugging implementation -----------------------------------

-// We will put a mark at the extreme end of each allocation block. We make

-// sure that we always allocate enough "extra memory" that we can fit in the

-// mark, and still provide the requested usable region. If ever that mark is

-// not as expected, then we know that the user is corrupting memory beyond their

-// request size, or that they have called free a second time without having

-// the memory allocated (again). This allows us to spot most double free()s,

-// but some can "slip by" or confuse our logic if the caller reallocates memory

-// (for a second use) before performing an evil double-free of a first

-// allocation

-// This code can be optimized, but for now, it is written to be most easily

-// understood, and flexible (since it is evolving a bit). Potential

-// optimizations include using other calculated data, such as class size, or

-// allocation size, which is known in the code above, but then is recalculated

-// below. Another potential optimization would be careful manual inlining of

-// code, but I *think* that the compile will probably do this for me, and I've

-// been careful to avoid aliasing issues that might make a compiler back-off.

-// Evolution includes experimenting with different marks, to minimize the chance

-// that a mark would be misunderstood (missed corruption). The marks are meant

-// to be hashed encoding of the location, so that they can't be copied over a

-// different region (by accident) without being detected (most of the time).

-// Enable the following define to turn on all the TCMalloc checking.

-// It will cost about 2% in performance, but it will catch double frees (most of

-// the time), and will often catch allocated-buffer overrun errors. This

-// validation is only active when TCMalloc is used as the allocator.

-#ifndef NDEBUG

-#define TCMALLOC_VALIDATION

-#endif

-#if !defined(TCMALLOC_VALIDATION)

-static size_t ExcludeSpaceForMark(size_t size) { return size; }

-static void AddRoomForMark(size_t* size) {}

-static void ExcludeMarkFromSize(size_t* new_size) {}

-static void MarkAllocatedRegion(void* ptr) {}

-static void ValidateAllocatedRegion(void* ptr, size_t cl) {}

-#else // TCMALLOC_VALIDATION

-static void DieFromDoubleFree() {

- char* p = NULL;

- p++;

- *p += 1; // Segv.

-static size_t DieFromBadFreePointer(void* unused) {

- char* p = NULL;

- p += 2;

- *p += 2; // Segv.

- return 0;

-static void DieFromMemoryCorruption() {

- char* p = NULL;

- p += 3;

- *p += 3; // Segv.

-// We can either do byte marking, or whole word marking based on the following

-// define. char is as small as we can get, and word marking probably provides

-// more than enough bits that we won't miss a corruption. Any sized integral

-// type can be used, but we just define two examples.

-// #define TCMALLOC_SMALL_VALIDATION

-#if defined (TCMALLOC_SMALL_VALIDATION)

-typedef char MarkType; // char saves memory... int is more complete.

-static const MarkType kAllocationMarkMask = static_cast<MarkType>(0x36);

-#else

-typedef int MarkType; // char saves memory... int is more complete.

-static const MarkType kAllocationMarkMask = static_cast<MarkType>(0xE1AB9536);

-#endif

-// TODO(jar): See if use of reference rather than pointer gets better inlining,

-// or if macro is needed. My fear is that taking address map preclude register

-// allocation :-(.

-inline static void AddRoomForMark(size_t* size) {

- *size += sizeof(kAllocationMarkMask);

-inline static void ExcludeMarkFromSize(size_t* new_size) {

- *new_size -= sizeof(kAllocationMarkMask);

-inline static size_t ExcludeSpaceForMark(size_t size) {

- return size - sizeof(kAllocationMarkMask); // Lie about size when asked.

-inline static MarkType* GetMarkLocation(void* ptr) {

- size_t class_size = GetSizeWithCallback(ptr, DieFromBadFreePointer);

- ASSERT(class_size % sizeof(kAllocationMarkMask) == 0);

- size_t last_index = (class_size / sizeof(kAllocationMarkMask)) - 1;

- return static_cast<MarkType*>(ptr) + last_index;

-// We hash in the mark location plus the pointer so that we effectively mix in

-// the size of the block. This means that if a span is used for different sizes

-// that the mark will be different. It would be good to hash in the size (which

-// we effectively get by using both mark location and pointer), but even better

-// would be to also include the class, as it concisely contains the entropy

-// found in the size (when we don't have large allocation), and there is less

-// risk of losing those bits to truncation. It would probably be good to combine

-// the high bits of size (capturing info about large blocks) with the class

-// (which is a 6 bit number).

-inline static MarkType GetMarkValue(void* ptr, MarkType* mark) {

- void* ptr2 = static_cast<void*>(mark);

- size_t offset1 = static_cast<char*>(ptr) - static_cast<char*>(NULL);

- size_t offset2 = static_cast<char*>(ptr2) - static_cast<char*>(NULL);

- static const int kInvariantBits = 2;

- ASSERT((offset1 >> kInvariantBits) << kInvariantBits == offset1);

- // Note: low bits of both offsets are invariants due to alignment. High bits

- // of both offsets are the same (unless we have a large allocation). Avoid

- // XORing high bits together, as they will cancel for most small allocations.

- MarkType ret = kAllocationMarkMask;

- // Using a little shift, we can safely XOR together both offsets.

- ret ^= static_cast<MarkType>(offset1 >> kInvariantBits) ^

- static_cast<MarkType>(offset2);

- if (sizeof(ret) == 1) {

- // Try to bring some high level bits into the mix.

- ret += static_cast<MarkType>(offset1 >> 8) ^

- static_cast<MarkType>(offset1 >> 16) ^

- static_cast<MarkType>(offset1 >> 24) ;

- }

- // Hash in high bits on a 64 bit architecture.

- if (sizeof(size_t) == 8 && sizeof(ret) == 4)

- ret += offset1 >> 16;

- if (ret == 0)

- ret = kAllocationMarkMask; // Avoid common pattern of all zeros.

- return ret;

-// TODO(jar): Use the passed in TCmalloc Class Index to calculate mark location

-// faster. The current implementation calls general functions, which have to

-// recalculate this in order to get the Class Size. This is a slow and wasteful

-// recomputation... but it is much more readable this way (for now).

-static void ValidateAllocatedRegion(void* ptr, size_t cl) {

- if (ptr == NULL) return;

- MarkType* mark = GetMarkLocation(ptr);

- MarkType allocated_mark = GetMarkValue(ptr, mark);

- MarkType current_mark = *mark;

- if (current_mark == ~allocated_mark)

- DieFromDoubleFree();

- if (current_mark != allocated_mark)

- DieFromMemoryCorruption();

-#ifndef NDEBUG

- // In debug mode, copy the mark into all the free'd region.

- size_t class_size = static_cast<size_t>(reinterpret_cast<char*>(mark) -

- reinterpret_cast<char*>(ptr));

- memset(ptr, static_cast<char>(0x36), class_size);

-#endif

- *mark = ~allocated_mark; // Distinctively not allocated.

-static void MarkAllocatedRegion(void* ptr) {

- if (ptr == NULL) return;

- MarkType* mark = GetMarkLocation(ptr);

- *mark = GetMarkValue(ptr, mark);

-#endif // TCMALLOC_VALIDATION