OLD | NEW |
1 // Copyright (c) 2005, 2007, Google Inc. | 1 // Copyright (c) 2005, 2007, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserv
ed. | 3 // Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserv
ed. |
4 // | 4 // |
5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
7 // met: | 7 // met: |
8 // | 8 // |
9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
80 #include "wtf/Assertions.h" | 80 #include "wtf/Assertions.h" |
81 #include "wtf/CPU.h" | 81 #include "wtf/CPU.h" |
82 #include "wtf/StdLibExtras.h" | 82 #include "wtf/StdLibExtras.h" |
83 #include "wtf/UnusedParam.h" | 83 #include "wtf/UnusedParam.h" |
84 | 84 |
85 #if OS(MACOSX) | 85 #if OS(MACOSX) |
86 #include <AvailabilityMacros.h> | 86 #include <AvailabilityMacros.h> |
87 #endif | 87 #endif |
88 | 88 |
89 #include <limits> | 89 #include <limits> |
90 #if OS(WINDOWS) | 90 #if OS(WIN) |
91 #include <windows.h> | 91 #include <windows.h> |
92 #else | 92 #else |
93 #include <pthread.h> | 93 #include <pthread.h> |
94 #endif | 94 #endif |
95 #include <stdlib.h> | 95 #include <stdlib.h> |
96 #include <string.h> | 96 #include <string.h> |
97 | 97 |
98 #ifndef NO_TCMALLOC_SAMPLES | 98 #ifndef NO_TCMALLOC_SAMPLES |
99 #define NO_TCMALLOC_SAMPLES | 99 #define NO_TCMALLOC_SAMPLES |
100 #endif | 100 #endif |
101 | 101 |
102 #if !USE(SYSTEM_MALLOC) && defined(NDEBUG) | 102 #if !USE(SYSTEM_MALLOC) && defined(NDEBUG) |
103 #define FORCE_SYSTEM_MALLOC 0 | 103 #define FORCE_SYSTEM_MALLOC 0 |
104 #else | 104 #else |
105 #define FORCE_SYSTEM_MALLOC 1 | 105 #define FORCE_SYSTEM_MALLOC 1 |
106 #endif | 106 #endif |
107 | 107 |
108 // Harden the pointers stored in the TCMalloc linked lists | 108 // Harden the pointers stored in the TCMalloc linked lists |
109 #if COMPILER(GCC) | 109 #if COMPILER(GCC) |
110 #define ENABLE_TCMALLOC_HARDENING 1 | 110 #define ENABLE_TCMALLOC_HARDENING 1 |
111 #endif | 111 #endif |
112 | 112 |
113 // Use a background thread to periodically scavenge memory to release back to th
e system | 113 // Use a background thread to periodically scavenge memory to release back to th
e system |
114 #define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 1 | 114 #define USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY 1 |
115 | 115 |
116 #ifndef NDEBUG | 116 #ifndef NDEBUG |
117 namespace WTF { | 117 namespace WTF { |
118 | 118 |
119 #if OS(WINDOWS) | 119 #if OS(WIN) |
120 | 120 |
121 // TLS_OUT_OF_INDEXES is not defined on WinCE. | 121 // TLS_OUT_OF_INDEXES is not defined on WinCE. |
122 #ifndef TLS_OUT_OF_INDEXES | 122 #ifndef TLS_OUT_OF_INDEXES |
123 #define TLS_OUT_OF_INDEXES 0xffffffff | 123 #define TLS_OUT_OF_INDEXES 0xffffffff |
124 #endif | 124 #endif |
125 | 125 |
126 static DWORD isForibiddenTlsIndex = TLS_OUT_OF_INDEXES; | 126 static DWORD isForibiddenTlsIndex = TLS_OUT_OF_INDEXES; |
127 static const LPVOID kTlsAllowValue = reinterpret_cast<LPVOID>(0); // Must be zer
o. | 127 static const LPVOID kTlsAllowValue = reinterpret_cast<LPVOID>(0); // Must be zer
o. |
128 static const LPVOID kTlsForbiddenValue = reinterpret_cast<LPVOID>(1); | 128 static const LPVOID kTlsForbiddenValue = reinterpret_cast<LPVOID>(1); |
129 | 129 |
(...skipping 14 matching lines...) Expand all Loading... |
144 TlsSetValue(isForibiddenTlsIndex, kTlsForbiddenValue); | 144 TlsSetValue(isForibiddenTlsIndex, kTlsForbiddenValue); |
145 } | 145 } |
146 | 146 |
147 void fastMallocAllow() | 147 void fastMallocAllow() |
148 { | 148 { |
149 if (isForibiddenTlsIndex == TLS_OUT_OF_INDEXES) | 149 if (isForibiddenTlsIndex == TLS_OUT_OF_INDEXES) |
150 return; | 150 return; |
151 TlsSetValue(isForibiddenTlsIndex, kTlsAllowValue); | 151 TlsSetValue(isForibiddenTlsIndex, kTlsAllowValue); |
152 } | 152 } |
153 | 153 |
154 #else // !OS(WINDOWS) | 154 #else // !OS(WIN) |
155 | 155 |
156 static pthread_key_t isForbiddenKey; | 156 static pthread_key_t isForbiddenKey; |
157 static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT; | 157 static pthread_once_t isForbiddenKeyOnce = PTHREAD_ONCE_INIT; |
158 static void initializeIsForbiddenKey() | 158 static void initializeIsForbiddenKey() |
159 { | 159 { |
160 pthread_key_create(&isForbiddenKey, 0); | 160 pthread_key_create(&isForbiddenKey, 0); |
161 } | 161 } |
162 | 162 |
163 #if !ASSERT_DISABLED | 163 #if !ASSERT_DISABLED |
164 static bool isForbidden() | 164 static bool isForbidden() |
165 { | 165 { |
166 pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); | 166 pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); |
167 return !!pthread_getspecific(isForbiddenKey); | 167 return !!pthread_getspecific(isForbiddenKey); |
168 } | 168 } |
169 #endif | 169 #endif |
170 | 170 |
171 void fastMallocForbid() | 171 void fastMallocForbid() |
172 { | 172 { |
173 pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); | 173 pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); |
174 pthread_setspecific(isForbiddenKey, &isForbiddenKey); | 174 pthread_setspecific(isForbiddenKey, &isForbiddenKey); |
175 } | 175 } |
176 | 176 |
177 void fastMallocAllow() | 177 void fastMallocAllow() |
178 { | 178 { |
179 pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); | 179 pthread_once(&isForbiddenKeyOnce, initializeIsForbiddenKey); |
180 pthread_setspecific(isForbiddenKey, 0); | 180 pthread_setspecific(isForbiddenKey, 0); |
181 } | 181 } |
182 #endif // OS(WINDOWS) | 182 #endif // OS(WIN) |
183 | 183 |
184 } // namespace WTF | 184 } // namespace WTF |
185 #endif // NDEBUG | 185 #endif // NDEBUG |
186 | 186 |
187 namespace WTF { | 187 namespace WTF { |
188 | 188 |
189 void* fastZeroedMalloc(size_t n) | 189 void* fastZeroedMalloc(size_t n) |
190 { | 190 { |
191 void* result = fastMalloc(n); | 191 void* result = fastMalloc(n); |
192 memset(result, 0, n); | 192 memset(result, 0, n); |
193 return result; | 193 return result; |
194 } | 194 } |
195 | 195 |
196 char* fastStrDup(const char* src) | 196 char* fastStrDup(const char* src) |
197 { | 197 { |
198 size_t len = strlen(src) + 1; | 198 size_t len = strlen(src) + 1; |
199 char* dup = static_cast<char*>(fastMalloc(len)); | 199 char* dup = static_cast<char*>(fastMalloc(len)); |
200 memcpy(dup, src, len); | 200 memcpy(dup, src, len); |
201 return dup; | 201 return dup; |
202 } | 202 } |
203 | 203 |
204 } // namespace WTF | 204 } // namespace WTF |
205 | 205 |
206 #if FORCE_SYSTEM_MALLOC | 206 #if FORCE_SYSTEM_MALLOC |
207 | 207 |
208 #if OS(MACOSX) | 208 #if OS(MACOSX) |
209 #include <malloc/malloc.h> | 209 #include <malloc/malloc.h> |
210 #elif OS(WINDOWS) | 210 #elif OS(WIN) |
211 #include <malloc.h> | 211 #include <malloc.h> |
212 #endif | 212 #endif |
213 | 213 |
214 namespace WTF { | 214 namespace WTF { |
215 | 215 |
216 size_t fastMallocGoodSize(size_t bytes) | 216 size_t fastMallocGoodSize(size_t bytes) |
217 { | 217 { |
218 #if OS(MACOSX) | 218 #if OS(MACOSX) |
219 return malloc_good_size(bytes); | 219 return malloc_good_size(bytes); |
220 #else | 220 #else |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
283 #include "TCSpinLock.h" | 283 #include "TCSpinLock.h" |
284 #include "TCSystemAlloc.h" | 284 #include "TCSystemAlloc.h" |
285 #include <algorithm> | 285 #include <algorithm> |
286 #include <errno.h> | 286 #include <errno.h> |
287 #include <pthread.h> | 287 #include <pthread.h> |
288 #include <stdarg.h> | 288 #include <stdarg.h> |
289 #include <stddef.h> | 289 #include <stddef.h> |
290 #if OS(UNIX) | 290 #if OS(UNIX) |
291 #include <unistd.h> | 291 #include <unistd.h> |
292 #endif | 292 #endif |
293 #if OS(WINDOWS) | 293 #if OS(WIN) |
294 #ifndef WIN32_LEAN_AND_MEAN | 294 #ifndef WIN32_LEAN_AND_MEAN |
295 #define WIN32_LEAN_AND_MEAN | 295 #define WIN32_LEAN_AND_MEAN |
296 #endif | 296 #endif |
297 #include <windows.h> | 297 #include <windows.h> |
298 #endif | 298 #endif |
299 | 299 |
300 #if OS(MACOSX) | 300 #if OS(MACOSX) |
301 #include "MallocZoneSupport.h" | 301 #include "MallocZoneSupport.h" |
302 #include "wtf/HashSet.h" | 302 #include "wtf/HashSet.h" |
303 #include "wtf/Vector.h" | 303 #include "wtf/Vector.h" |
(...skipping 1289 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1593 #if OS(MACOSX) | 1593 #if OS(MACOSX) |
1594 friend class FastMallocZone; | 1594 friend class FastMallocZone; |
1595 #endif | 1595 #endif |
1596 | 1596 |
1597 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY | 1597 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY |
1598 void initializeScavenger(); | 1598 void initializeScavenger(); |
1599 ALWAYS_INLINE void signalScavenger(); | 1599 ALWAYS_INLINE void signalScavenger(); |
1600 void scavenge(); | 1600 void scavenge(); |
1601 ALWAYS_INLINE bool shouldScavenge() const; | 1601 ALWAYS_INLINE bool shouldScavenge() const; |
1602 | 1602 |
1603 #if HAVE(DISPATCH_H) || OS(WINDOWS) | 1603 #if HAVE(DISPATCH_H) || OS(WIN) |
1604 void periodicScavenge(); | 1604 void periodicScavenge(); |
1605 ALWAYS_INLINE bool isScavengerSuspended(); | 1605 ALWAYS_INLINE bool isScavengerSuspended(); |
1606 ALWAYS_INLINE void scheduleScavenger(); | 1606 ALWAYS_INLINE void scheduleScavenger(); |
1607 ALWAYS_INLINE void rescheduleScavenger(); | 1607 ALWAYS_INLINE void rescheduleScavenger(); |
1608 ALWAYS_INLINE void suspendScavenger(); | 1608 ALWAYS_INLINE void suspendScavenger(); |
1609 #endif | 1609 #endif |
1610 | 1610 |
1611 #if HAVE(DISPATCH_H) | 1611 #if HAVE(DISPATCH_H) |
1612 dispatch_queue_t m_scavengeQueue; | 1612 dispatch_queue_t m_scavengeQueue; |
1613 dispatch_source_t m_scavengeTimer; | 1613 dispatch_source_t m_scavengeTimer; |
1614 bool m_scavengingSuspended; | 1614 bool m_scavengingSuspended; |
1615 #elif OS(WINDOWS) | 1615 #elif OS(WIN) |
1616 static void CALLBACK scavengerTimerFired(void*, BOOLEAN); | 1616 static void CALLBACK scavengerTimerFired(void*, BOOLEAN); |
1617 HANDLE m_scavengeQueueTimer; | 1617 HANDLE m_scavengeQueueTimer; |
1618 #else | 1618 #else |
1619 static NO_RETURN_WITH_VALUE void* runScavengerThread(void*); | 1619 static NO_RETURN_WITH_VALUE void* runScavengerThread(void*); |
1620 NO_RETURN void scavengerThread(); | 1620 NO_RETURN void scavengerThread(); |
1621 | 1621 |
1622 // Keeps track of whether the background thread is actively scavenging memory
every kScavengeDelayInSeconds, or | 1622 // Keeps track of whether the background thread is actively scavenging memory
every kScavengeDelayInSeconds, or |
1623 // it's blocked waiting for more pages to be deleted. | 1623 // it's blocked waiting for more pages to be deleted. |
1624 bool m_scavengeThreadActive; | 1624 bool m_scavengeThreadActive; |
1625 | 1625 |
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1692 // Nothing to do here for libdispatch. | 1692 // Nothing to do here for libdispatch. |
1693 } | 1693 } |
1694 | 1694 |
1695 ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger() | 1695 ALWAYS_INLINE void TCMalloc_PageHeap::suspendScavenger() |
1696 { | 1696 { |
1697 ASSERT(pageheap_lock.IsHeld()); | 1697 ASSERT(pageheap_lock.IsHeld()); |
1698 m_scavengingSuspended = true; | 1698 m_scavengingSuspended = true; |
1699 dispatch_suspend(m_scavengeTimer); | 1699 dispatch_suspend(m_scavengeTimer); |
1700 } | 1700 } |
1701 | 1701 |
1702 #elif OS(WINDOWS) | 1702 #elif OS(WIN) |
1703 | 1703 |
1704 void TCMalloc_PageHeap::scavengerTimerFired(void* context, BOOLEAN) | 1704 void TCMalloc_PageHeap::scavengerTimerFired(void* context, BOOLEAN) |
1705 { | 1705 { |
1706 static_cast<TCMalloc_PageHeap*>(context)->periodicScavenge(); | 1706 static_cast<TCMalloc_PageHeap*>(context)->periodicScavenge(); |
1707 } | 1707 } |
1708 | 1708 |
1709 void TCMalloc_PageHeap::initializeScavenger() | 1709 void TCMalloc_PageHeap::initializeScavenger() |
1710 { | 1710 { |
1711 m_scavengeQueueTimer = 0; | 1711 m_scavengeQueueTimer = 0; |
1712 } | 1712 } |
(...skipping 650 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2363 } | 2363 } |
2364 }; | 2364 }; |
2365 | 2365 |
2366 //------------------------------------------------------------------- | 2366 //------------------------------------------------------------------- |
2367 // Data kept per thread | 2367 // Data kept per thread |
2368 //------------------------------------------------------------------- | 2368 //------------------------------------------------------------------- |
2369 | 2369 |
2370 class TCMalloc_ThreadCache { | 2370 class TCMalloc_ThreadCache { |
2371 private: | 2371 private: |
2372 typedef TCMalloc_ThreadCache_FreeList FreeList; | 2372 typedef TCMalloc_ThreadCache_FreeList FreeList; |
2373 #if OS(WINDOWS) | 2373 #if OS(WIN) |
2374 typedef DWORD ThreadIdentifier; | 2374 typedef DWORD ThreadIdentifier; |
2375 #else | 2375 #else |
2376 typedef pthread_t ThreadIdentifier; | 2376 typedef pthread_t ThreadIdentifier; |
2377 #endif | 2377 #endif |
2378 | 2378 |
2379 size_t size_; // Combined size of data | 2379 size_t size_; // Combined size of data |
2380 ThreadIdentifier tid_; // Which thread owns it | 2380 ThreadIdentifier tid_; // Which thread owns it |
2381 bool in_setspecific_; // Called pthread_setspecific? | 2381 bool in_setspecific_; // Called pthread_setspecific? |
2382 FreeList list_[kNumClasses]; // Array indexed by size-class | 2382 FreeList list_[kNumClasses]; // Array indexed by size-class |
2383 | 2383 |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2468 | 2468 |
2469 size_t fastMallocGoodSize(size_t bytes) | 2469 size_t fastMallocGoodSize(size_t bytes) |
2470 { | 2470 { |
2471 if (!phinited) | 2471 if (!phinited) |
2472 TCMalloc_ThreadCache::InitModule(); | 2472 TCMalloc_ThreadCache::InitModule(); |
2473 return AllocationSize(bytes); | 2473 return AllocationSize(bytes); |
2474 } | 2474 } |
2475 | 2475 |
2476 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY | 2476 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY |
2477 | 2477 |
2478 #if HAVE(DISPATCH_H) || OS(WINDOWS) | 2478 #if HAVE(DISPATCH_H) || OS(WIN) |
2479 | 2479 |
2480 void TCMalloc_PageHeap::periodicScavenge() | 2480 void TCMalloc_PageHeap::periodicScavenge() |
2481 { | 2481 { |
2482 SpinLockHolder h(&pageheap_lock); | 2482 SpinLockHolder h(&pageheap_lock); |
2483 pageheap->scavenge(); | 2483 pageheap->scavenge(); |
2484 | 2484 |
2485 if (shouldScavenge()) { | 2485 if (shouldScavenge()) { |
2486 rescheduleScavenger(); | 2486 rescheduleScavenger(); |
2487 return; | 2487 return; |
2488 } | 2488 } |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2550 #ifdef HAVE_TLS | 2550 #ifdef HAVE_TLS |
2551 static __thread TCMalloc_ThreadCache *threadlocal_heap; | 2551 static __thread TCMalloc_ThreadCache *threadlocal_heap; |
2552 #endif | 2552 #endif |
2553 // Thread-specific key. Initialization here is somewhat tricky | 2553 // Thread-specific key. Initialization here is somewhat tricky |
2554 // because some Linux startup code invokes malloc() before it | 2554 // because some Linux startup code invokes malloc() before it |
2555 // is in a good enough state to handle pthread_keycreate(). | 2555 // is in a good enough state to handle pthread_keycreate(). |
2556 // Therefore, we use TSD keys only after tsd_inited is set to true. | 2556 // Therefore, we use TSD keys only after tsd_inited is set to true. |
2557 // Until then, we use a slow path to get the heap object. | 2557 // Until then, we use a slow path to get the heap object. |
2558 static bool tsd_inited = false; | 2558 static bool tsd_inited = false; |
2559 static pthread_key_t heap_key; | 2559 static pthread_key_t heap_key; |
2560 #if OS(WINDOWS) | 2560 #if OS(WIN) |
2561 DWORD tlsIndex = TLS_OUT_OF_INDEXES; | 2561 DWORD tlsIndex = TLS_OUT_OF_INDEXES; |
2562 #endif | 2562 #endif |
2563 | 2563 |
2564 static ALWAYS_INLINE void setThreadHeap(TCMalloc_ThreadCache* heap) | 2564 static ALWAYS_INLINE void setThreadHeap(TCMalloc_ThreadCache* heap) |
2565 { | 2565 { |
2566 #if USE(PTHREAD_GETSPECIFIC_DIRECT) | 2566 #if USE(PTHREAD_GETSPECIFIC_DIRECT) |
2567 // Can't have two libraries both doing this in the same process, | 2567 // Can't have two libraries both doing this in the same process, |
2568 // so check and make this crash right away. | 2568 // so check and make this crash right away. |
2569 if (pthread_getspecific(heap_key)) | 2569 if (pthread_getspecific(heap_key)) |
2570 CRASH(); | 2570 CRASH(); |
2571 #endif | 2571 #endif |
2572 | 2572 |
2573 // Still do pthread_setspecific even if there's an alternate form | 2573 // Still do pthread_setspecific even if there's an alternate form |
2574 // of thread-local storage in use, to benefit from the delete callback. | 2574 // of thread-local storage in use, to benefit from the delete callback. |
2575 pthread_setspecific(heap_key, heap); | 2575 pthread_setspecific(heap_key, heap); |
2576 | 2576 |
2577 #if OS(WINDOWS) | 2577 #if OS(WIN) |
2578 TlsSetValue(tlsIndex, heap); | 2578 TlsSetValue(tlsIndex, heap); |
2579 #endif | 2579 #endif |
2580 } | 2580 } |
2581 | 2581 |
2582 // Allocator for thread heaps | 2582 // Allocator for thread heaps |
2583 static PageHeapAllocator<TCMalloc_ThreadCache> threadheap_allocator; | 2583 static PageHeapAllocator<TCMalloc_ThreadCache> threadheap_allocator; |
2584 | 2584 |
2585 // Linked list of heap objects. Protected by pageheap_lock. | 2585 // Linked list of heap objects. Protected by pageheap_lock. |
2586 static TCMalloc_ThreadCache* thread_heaps = NULL; | 2586 static TCMalloc_ThreadCache* thread_heaps = NULL; |
2587 static int thread_heap_count = 0; | 2587 static int thread_heap_count = 0; |
(...skipping 237 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2825 lock_.Unlock(); | 2825 lock_.Unlock(); |
2826 const size_t npages = class_to_pages[size_class_]; | 2826 const size_t npages = class_to_pages[size_class_]; |
2827 | 2827 |
2828 Span* span; | 2828 Span* span; |
2829 { | 2829 { |
2830 SpinLockHolder h(&pageheap_lock); | 2830 SpinLockHolder h(&pageheap_lock); |
2831 span = pageheap->New(npages); | 2831 span = pageheap->New(npages); |
2832 if (span) pageheap->RegisterSizeClass(span, size_class_); | 2832 if (span) pageheap->RegisterSizeClass(span, size_class_); |
2833 } | 2833 } |
2834 if (span == NULL) { | 2834 if (span == NULL) { |
2835 #if OS(WINDOWS) | 2835 #if OS(WIN) |
2836 MESSAGE("allocation failed: %d\n", ::GetLastError()); | 2836 MESSAGE("allocation failed: %d\n", ::GetLastError()); |
2837 #else | 2837 #else |
2838 MESSAGE("allocation failed: %d\n", errno); | 2838 MESSAGE("allocation failed: %d\n", errno); |
2839 #endif | 2839 #endif |
2840 lock_.Lock(); | 2840 lock_.Lock(); |
2841 return; | 2841 return; |
2842 } | 2842 } |
2843 ASSERT_SPAN_COMMITTED(span); | 2843 ASSERT_SPAN_COMMITTED(span); |
2844 ASSERT(span->length == npages); | 2844 ASSERT(span->length == npages); |
2845 // Cache sizeclass info eagerly. Locking is not necessary. | 2845 // Cache sizeclass info eagerly. Locking is not necessary. |
(...skipping 264 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3110 thread_heap_count++; | 3110 thread_heap_count++; |
3111 RecomputeThreadCacheSize(); | 3111 RecomputeThreadCacheSize(); |
3112 return heap; | 3112 return heap; |
3113 } | 3113 } |
3114 | 3114 |
3115 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetThreadHeap() { | 3115 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetThreadHeap() { |
3116 #ifdef HAVE_TLS | 3116 #ifdef HAVE_TLS |
3117 // __thread is faster, but only when the kernel supports it | 3117 // __thread is faster, but only when the kernel supports it |
3118 if (KernelSupportsTLS()) | 3118 if (KernelSupportsTLS()) |
3119 return threadlocal_heap; | 3119 return threadlocal_heap; |
3120 #elif OS(WINDOWS) | 3120 #elif OS(WIN) |
3121 return static_cast<TCMalloc_ThreadCache*>(TlsGetValue(tlsIndex)); | 3121 return static_cast<TCMalloc_ThreadCache*>(TlsGetValue(tlsIndex)); |
3122 #else | 3122 #else |
3123 return static_cast<TCMalloc_ThreadCache*>(pthread_getspecific(heap_key)); | 3123 return static_cast<TCMalloc_ThreadCache*>(pthread_getspecific(heap_key)); |
3124 #endif | 3124 #endif |
3125 } | 3125 } |
3126 | 3126 |
3127 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetCache() { | 3127 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::GetCache() { |
3128 TCMalloc_ThreadCache* ptr = NULL; | 3128 TCMalloc_ThreadCache* ptr = NULL; |
3129 if (!tsd_inited) { | 3129 if (!tsd_inited) { |
3130 InitModule(); | 3130 InitModule(); |
(...skipping 13 matching lines...) Expand all Loading... |
3144 return reinterpret_cast<TCMalloc_ThreadCache*>(p); | 3144 return reinterpret_cast<TCMalloc_ThreadCache*>(p); |
3145 } | 3145 } |
3146 | 3146 |
3147 void TCMalloc_ThreadCache::InitTSD() { | 3147 void TCMalloc_ThreadCache::InitTSD() { |
3148 ASSERT(!tsd_inited); | 3148 ASSERT(!tsd_inited); |
3149 #if USE(PTHREAD_GETSPECIFIC_DIRECT) | 3149 #if USE(PTHREAD_GETSPECIFIC_DIRECT) |
3150 pthread_key_init_np(heap_key, DestroyThreadCache); | 3150 pthread_key_init_np(heap_key, DestroyThreadCache); |
3151 #else | 3151 #else |
3152 pthread_key_create(&heap_key, DestroyThreadCache); | 3152 pthread_key_create(&heap_key, DestroyThreadCache); |
3153 #endif | 3153 #endif |
3154 #if OS(WINDOWS) | 3154 #if OS(WIN) |
3155 tlsIndex = TlsAlloc(); | 3155 tlsIndex = TlsAlloc(); |
3156 #endif | 3156 #endif |
3157 tsd_inited = true; | 3157 tsd_inited = true; |
3158 | 3158 |
3159 #if !OS(WINDOWS) | 3159 #if !OS(WIN) |
3160 // We may have used a fake pthread_t for the main thread. Fix it. | 3160 // We may have used a fake pthread_t for the main thread. Fix it. |
3161 pthread_t zero; | 3161 pthread_t zero; |
3162 memset(&zero, 0, sizeof(zero)); | 3162 memset(&zero, 0, sizeof(zero)); |
3163 #endif | 3163 #endif |
3164 ASSERT(pageheap_lock.IsHeld()); | 3164 ASSERT(pageheap_lock.IsHeld()); |
3165 for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) { | 3165 for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) { |
3166 #if OS(WINDOWS) | 3166 #if OS(WIN) |
3167 if (h->tid_ == 0) { | 3167 if (h->tid_ == 0) { |
3168 h->tid_ = GetCurrentThreadId(); | 3168 h->tid_ = GetCurrentThreadId(); |
3169 } | 3169 } |
3170 #else | 3170 #else |
3171 if (pthread_equal(h->tid_, zero)) { | 3171 if (pthread_equal(h->tid_, zero)) { |
3172 h->tid_ = pthread_self(); | 3172 h->tid_ = pthread_self(); |
3173 } | 3173 } |
3174 #endif | 3174 #endif |
3175 } | 3175 } |
3176 } | 3176 } |
3177 | 3177 |
3178 TCMalloc_ThreadCache* TCMalloc_ThreadCache::CreateCacheIfNecessary() { | 3178 TCMalloc_ThreadCache* TCMalloc_ThreadCache::CreateCacheIfNecessary() { |
3179 // Initialize per-thread data if necessary | 3179 // Initialize per-thread data if necessary |
3180 TCMalloc_ThreadCache* heap = NULL; | 3180 TCMalloc_ThreadCache* heap = NULL; |
3181 { | 3181 { |
3182 SpinLockHolder h(&pageheap_lock); | 3182 SpinLockHolder h(&pageheap_lock); |
3183 | 3183 |
3184 #if OS(WINDOWS) | 3184 #if OS(WIN) |
3185 DWORD me; | 3185 DWORD me; |
3186 if (!tsd_inited) { | 3186 if (!tsd_inited) { |
3187 me = 0; | 3187 me = 0; |
3188 } else { | 3188 } else { |
3189 me = GetCurrentThreadId(); | 3189 me = GetCurrentThreadId(); |
3190 } | 3190 } |
3191 #else | 3191 #else |
3192 // Early on in glibc's life, we cannot even call pthread_self() | 3192 // Early on in glibc's life, we cannot even call pthread_self() |
3193 pthread_t me; | 3193 pthread_t me; |
3194 if (!tsd_inited) { | 3194 if (!tsd_inited) { |
3195 memset(&me, 0, sizeof(me)); | 3195 memset(&me, 0, sizeof(me)); |
3196 } else { | 3196 } else { |
3197 me = pthread_self(); | 3197 me = pthread_self(); |
3198 } | 3198 } |
3199 #endif | 3199 #endif |
3200 | 3200 |
3201 // This may be a recursive malloc call from pthread_setspecific() | 3201 // This may be a recursive malloc call from pthread_setspecific() |
3202 // In that case, the heap for this thread has already been created | 3202 // In that case, the heap for this thread has already been created |
3203 // and added to the linked list. So we search for that first. | 3203 // and added to the linked list. So we search for that first. |
3204 for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) { | 3204 for (TCMalloc_ThreadCache* h = thread_heaps; h != NULL; h = h->next_) { |
3205 #if OS(WINDOWS) | 3205 #if OS(WIN) |
3206 if (h->tid_ == me) { | 3206 if (h->tid_ == me) { |
3207 #else | 3207 #else |
3208 if (pthread_equal(h->tid_, me)) { | 3208 if (pthread_equal(h->tid_, me)) { |
3209 #endif | 3209 #endif |
3210 heap = h; | 3210 heap = h; |
3211 break; | 3211 break; |
3212 } | 3212 } |
3213 } | 3213 } |
3214 | 3214 |
3215 if (heap == NULL) heap = NewHeap(me, HARDENING_ENTROPY); | 3215 if (heap == NULL) heap = NewHeap(me, HARDENING_ENTROPY); |
(...skipping 675 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3891 void FastMallocZone::init() | 3891 void FastMallocZone::init() |
3892 { | 3892 { |
3893 static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Cen
tral_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); | 3893 static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Cen
tral_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); |
3894 } | 3894 } |
3895 | 3895 |
3896 #endif // OS(MACOSX) | 3896 #endif // OS(MACOSX) |
3897 | 3897 |
3898 } // namespace WTF | 3898 } // namespace WTF |
3899 | 3899 |
3900 #endif // FORCE_SYSTEM_MALLOC | 3900 #endif // FORCE_SYSTEM_MALLOC |
OLD | NEW |