OLD | NEW |
1 // Copyright (c) 2005, 2007, Google Inc. | 1 // Copyright (c) 2005, 2007, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserv
ed. | 3 // Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserv
ed. |
4 // | 4 // |
5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
7 // met: | 7 // met: |
8 // | 8 // |
9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
75 // goes from about 1100 ns to about 300 ns. | 75 // goes from about 1100 ns to about 300 ns. |
76 | 76 |
77 #include "config.h" | 77 #include "config.h" |
78 #include "wtf/FastMalloc.h" | 78 #include "wtf/FastMalloc.h" |
79 | 79 |
80 #include "wtf/Assertions.h" | 80 #include "wtf/Assertions.h" |
81 #include "wtf/CPU.h" | 81 #include "wtf/CPU.h" |
82 #include "wtf/StdLibExtras.h" | 82 #include "wtf/StdLibExtras.h" |
83 #include "wtf/UnusedParam.h" | 83 #include "wtf/UnusedParam.h" |
84 | 84 |
85 #if OS(DARWIN) | 85 #if OS(MACOSX) |
86 #include <AvailabilityMacros.h> | 86 #include <AvailabilityMacros.h> |
87 #endif | 87 #endif |
88 | 88 |
89 #include <limits> | 89 #include <limits> |
90 #if OS(WINDOWS) | 90 #if OS(WINDOWS) |
91 #include <windows.h> | 91 #include <windows.h> |
92 #else | 92 #else |
93 #include <pthread.h> | 93 #include <pthread.h> |
94 #endif | 94 #endif |
95 #include <stdlib.h> | 95 #include <stdlib.h> |
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
198 size_t len = strlen(src) + 1; | 198 size_t len = strlen(src) + 1; |
199 char* dup = static_cast<char*>(fastMalloc(len)); | 199 char* dup = static_cast<char*>(fastMalloc(len)); |
200 memcpy(dup, src, len); | 200 memcpy(dup, src, len); |
201 return dup; | 201 return dup; |
202 } | 202 } |
203 | 203 |
204 } // namespace WTF | 204 } // namespace WTF |
205 | 205 |
206 #if FORCE_SYSTEM_MALLOC | 206 #if FORCE_SYSTEM_MALLOC |
207 | 207 |
208 #if OS(DARWIN) | 208 #if OS(MACOSX) |
209 #include <malloc/malloc.h> | 209 #include <malloc/malloc.h> |
210 #elif OS(WINDOWS) | 210 #elif OS(WINDOWS) |
211 #include <malloc.h> | 211 #include <malloc.h> |
212 #endif | 212 #endif |
213 | 213 |
214 namespace WTF { | 214 namespace WTF { |
215 | 215 |
216 size_t fastMallocGoodSize(size_t bytes) | 216 size_t fastMallocGoodSize(size_t bytes) |
217 { | 217 { |
218 #if OS(DARWIN) | 218 #if OS(MACOSX) |
219 return malloc_good_size(bytes); | 219 return malloc_good_size(bytes); |
220 #else | 220 #else |
221 return bytes; | 221 return bytes; |
222 #endif | 222 #endif |
223 } | 223 } |
224 | 224 |
225 void* fastMalloc(size_t n) | 225 void* fastMalloc(size_t n) |
226 { | 226 { |
227 ASSERT(!isForbidden()); | 227 ASSERT(!isForbidden()); |
228 | 228 |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
262 void releaseFastMallocFreeMemory() { } | 262 void releaseFastMallocFreeMemory() { } |
263 | 263 |
264 FastMallocStatistics fastMallocStatistics() | 264 FastMallocStatistics fastMallocStatistics() |
265 { | 265 { |
266 FastMallocStatistics statistics = { 0, 0, 0 }; | 266 FastMallocStatistics statistics = { 0, 0, 0 }; |
267 return statistics; | 267 return statistics; |
268 } | 268 } |
269 | 269 |
270 } // namespace WTF | 270 } // namespace WTF |
271 | 271 |
272 #if OS(DARWIN) | 272 #if OS(MACOSX) |
273 // This symbol is present in the JavaScriptCore exports file even when FastMallo
c is disabled. | 273 // This symbol is present in the JavaScriptCore exports file even when FastMallo
c is disabled. |
274 // It will never be used in this case, so it's type and value are less interesti
ng than its presence. | 274 // It will never be used in this case, so it's type and value are less interesti
ng than its presence. |
275 extern "C" const int jscore_fastmalloc_introspection = 0; | 275 extern "C" const int jscore_fastmalloc_introspection = 0; |
276 #endif | 276 #endif |
277 | 277 |
278 #else // FORCE_SYSTEM_MALLOC | 278 #else // FORCE_SYSTEM_MALLOC |
279 | 279 |
280 #include "Compiler.h" | 280 #include "Compiler.h" |
281 #include "TCPackedCache.h" | 281 #include "TCPackedCache.h" |
282 #include "TCPageMap.h" | 282 #include "TCPageMap.h" |
283 #include "TCSpinLock.h" | 283 #include "TCSpinLock.h" |
284 #include "TCSystemAlloc.h" | 284 #include "TCSystemAlloc.h" |
285 #include <algorithm> | 285 #include <algorithm> |
286 #include <errno.h> | 286 #include <errno.h> |
287 #include <pthread.h> | 287 #include <pthread.h> |
288 #include <stdarg.h> | 288 #include <stdarg.h> |
289 #include <stddef.h> | 289 #include <stddef.h> |
290 #if OS(UNIX) | 290 #if OS(UNIX) |
291 #include <unistd.h> | 291 #include <unistd.h> |
292 #endif | 292 #endif |
293 #if OS(WINDOWS) | 293 #if OS(WINDOWS) |
294 #ifndef WIN32_LEAN_AND_MEAN | 294 #ifndef WIN32_LEAN_AND_MEAN |
295 #define WIN32_LEAN_AND_MEAN | 295 #define WIN32_LEAN_AND_MEAN |
296 #endif | 296 #endif |
297 #include <windows.h> | 297 #include <windows.h> |
298 #endif | 298 #endif |
299 | 299 |
300 #if OS(DARWIN) | 300 #if OS(MACOSX) |
301 #include "MallocZoneSupport.h" | 301 #include "MallocZoneSupport.h" |
302 #include "wtf/HashSet.h" | 302 #include "wtf/HashSet.h" |
303 #include "wtf/Vector.h" | 303 #include "wtf/Vector.h" |
304 #else | 304 #else |
305 #include "wtf/CurrentTime.h" | 305 #include "wtf/CurrentTime.h" |
306 #endif | 306 #endif |
307 | 307 |
308 #if HAVE(DISPATCH_H) | 308 #if HAVE(DISPATCH_H) |
309 #include <dispatch/dispatch.h> | 309 #include <dispatch/dispatch.h> |
310 #endif | 310 #endif |
311 | 311 |
312 #ifdef __has_include | 312 #ifdef __has_include |
313 #if __has_include(<System/pthread_machdep.h>) | 313 #if __has_include(<System/pthread_machdep.h>) |
314 | 314 |
315 #include <System/pthread_machdep.h> | 315 #include <System/pthread_machdep.h> |
316 | 316 |
317 #endif | 317 #endif |
318 #endif | 318 #endif |
319 | 319 |
320 #ifndef PRIuS | 320 #ifndef PRIuS |
321 #define PRIuS "zu" | 321 #define PRIuS "zu" |
322 #endif | 322 #endif |
323 | 323 |
324 // Calling pthread_getspecific through a global function pointer is faster than
a normal | 324 // Calling pthread_getspecific through a global function pointer is faster than
a normal |
325 // call to the function on Mac OS X, and it's used in performance-critical code.
So we | 325 // call to the function on Mac OS X, and it's used in performance-critical code.
So we |
326 // use a function pointer. But that's not necessarily faster on other platforms,
and we had | 326 // use a function pointer. But that's not necessarily faster on other platforms,
and we had |
327 // problems with this technique on Windows, so we'll do this only on Mac OS X. | 327 // problems with this technique on Windows, so we'll do this only on Mac OS X. |
328 #if OS(DARWIN) | 328 #if OS(MACOSX) |
329 #if !USE(PTHREAD_GETSPECIFIC_DIRECT) | 329 #if !USE(PTHREAD_GETSPECIFIC_DIRECT) |
330 static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_ge
tspecific; | 330 static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_ge
tspecific; |
331 #define pthread_getspecific(key) pthread_getspecific_function_pointer(key) | 331 #define pthread_getspecific(key) pthread_getspecific_function_pointer(key) |
332 #else | 332 #else |
333 #define pthread_getspecific(key) _pthread_getspecific_direct(key) | 333 #define pthread_getspecific(key) _pthread_getspecific_direct(key) |
334 #define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val)) | 334 #define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val)) |
335 #endif | 335 #endif |
336 #endif | 336 #endif |
337 | 337 |
338 #define DEFINE_VARIABLE(type, name, value, meaning) \ | 338 #define DEFINE_VARIABLE(type, name, value, meaning) \ |
(...skipping 17 matching lines...) Expand all Loading... |
356 #define realloc fastRealloc | 356 #define realloc fastRealloc |
357 | 357 |
358 #define MESSAGE LOG_ERROR | 358 #define MESSAGE LOG_ERROR |
359 #define CHECK_CONDITION ASSERT | 359 #define CHECK_CONDITION ASSERT |
360 | 360 |
361 static const char kLLHardeningMask = 0; | 361 static const char kLLHardeningMask = 0; |
362 template <unsigned> struct EntropySource; | 362 template <unsigned> struct EntropySource; |
363 template <> struct EntropySource<4> { | 363 template <> struct EntropySource<4> { |
364 static uint32_t value() | 364 static uint32_t value() |
365 { | 365 { |
366 #if OS(DARWIN) | 366 #if OS(MACOSX) |
367 return arc4random(); | 367 return arc4random(); |
368 #else | 368 #else |
369 return static_cast<uint32_t>(static_cast<uintptr_t>(currentTime() * 1000
0) ^ reinterpret_cast<uintptr_t>(&kLLHardeningMask)); | 369 return static_cast<uint32_t>(static_cast<uintptr_t>(currentTime() * 1000
0) ^ reinterpret_cast<uintptr_t>(&kLLHardeningMask)); |
370 #endif | 370 #endif |
371 } | 371 } |
372 }; | 372 }; |
373 | 373 |
374 template <> struct EntropySource<8> { | 374 template <> struct EntropySource<8> { |
375 static uint64_t value() | 375 static uint64_t value() |
376 { | 376 { |
(...skipping 554 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
931 | 931 |
932 void Delete(T* p) { | 932 void Delete(T* p) { |
933 HardenedSLL new_head = HardenedSLL::create(p); | 933 HardenedSLL new_head = HardenedSLL::create(p); |
934 SLL_SetNext(new_head, free_list_, entropy_); | 934 SLL_SetNext(new_head, free_list_, entropy_); |
935 free_list_ = new_head; | 935 free_list_ = new_head; |
936 inuse_--; | 936 inuse_--; |
937 } | 937 } |
938 | 938 |
939 int inuse() const { return inuse_; } | 939 int inuse() const { return inuse_; } |
940 | 940 |
941 #if OS(DARWIN) | 941 #if OS(MACOSX) |
942 template <class Recorder> | 942 template <class Recorder> |
943 void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader&
reader) | 943 void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader&
reader) |
944 { | 944 { |
945 for (HardenedSLL adminAllocation = allocated_regions_; adminAllocation; ad
minAllocation.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<voi
d**>(adminAllocation.value()), entropy_))) | 945 for (HardenedSLL adminAllocation = allocated_regions_; adminAllocation; ad
minAllocation.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<voi
d**>(adminAllocation.value()), entropy_))) |
946 recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation.v
alue()), kAllocIncrement); | 946 recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation.v
alue()), kAllocIncrement); |
947 } | 947 } |
948 #endif | 948 #endif |
949 }; | 949 }; |
950 | 950 |
951 // ------------------------------------------------------------------------- | 951 // ------------------------------------------------------------------------- |
(...skipping 313 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1265 // to be exactly 64 bytes. | 1265 // to be exactly 64 bytes. |
1266 template <> class TCMalloc_Central_FreeListPadded_Template<0> : public TCMalloc_
Central_FreeList { | 1266 template <> class TCMalloc_Central_FreeListPadded_Template<0> : public TCMalloc_
Central_FreeList { |
1267 }; | 1267 }; |
1268 | 1268 |
1269 typedef TCMalloc_Central_FreeListPadded_Template<sizeof(TCMalloc_Central_FreeLis
t) % 64> TCMalloc_Central_FreeListPadded; | 1269 typedef TCMalloc_Central_FreeListPadded_Template<sizeof(TCMalloc_Central_FreeLis
t) % 64> TCMalloc_Central_FreeListPadded; |
1270 | 1270 |
1271 #if COMPILER(CLANG) && defined(__has_warning) | 1271 #if COMPILER(CLANG) && defined(__has_warning) |
1272 #pragma clang diagnostic pop | 1272 #pragma clang diagnostic pop |
1273 #endif | 1273 #endif |
1274 | 1274 |
1275 #if OS(DARWIN) | 1275 #if OS(MACOSX) |
1276 struct Span; | 1276 struct Span; |
1277 class TCMalloc_PageHeap; | 1277 class TCMalloc_PageHeap; |
1278 class TCMalloc_ThreadCache; | 1278 class TCMalloc_ThreadCache; |
1279 template <typename T> class PageHeapAllocator; | 1279 template <typename T> class PageHeapAllocator; |
1280 | 1280 |
1281 class FastMallocZone { | 1281 class FastMallocZone { |
1282 public: | 1282 public: |
1283 static void init(); | 1283 static void init(); |
1284 | 1284 |
1285 static kern_return_t enumerate(task_t, void*, unsigned typeMmask, vm_address
_t zoneAddress, memory_reader_t, vm_range_recorder_t); | 1285 static kern_return_t enumerate(task_t, void*, unsigned typeMmask, vm_address
_t zoneAddress, memory_reader_t, vm_range_recorder_t); |
(...skipping 297 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1583 // IncrementalScavenge(n) is called whenever n pages are freed. | 1583 // IncrementalScavenge(n) is called whenever n pages are freed. |
1584 void IncrementalScavenge(Length n); | 1584 void IncrementalScavenge(Length n); |
1585 #endif | 1585 #endif |
1586 | 1586 |
1587 // Number of pages to deallocate before doing more scavenging | 1587 // Number of pages to deallocate before doing more scavenging |
1588 int64_t scavenge_counter_; | 1588 int64_t scavenge_counter_; |
1589 | 1589 |
1590 // Index of last free list we scavenged | 1590 // Index of last free list we scavenged |
1591 size_t scavenge_index_; | 1591 size_t scavenge_index_; |
1592 | 1592 |
1593 #if OS(DARWIN) | 1593 #if OS(MACOSX) |
1594 friend class FastMallocZone; | 1594 friend class FastMallocZone; |
1595 #endif | 1595 #endif |
1596 | 1596 |
1597 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY | 1597 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY |
1598 void initializeScavenger(); | 1598 void initializeScavenger(); |
1599 ALWAYS_INLINE void signalScavenger(); | 1599 ALWAYS_INLINE void signalScavenger(); |
1600 void scavenge(); | 1600 void scavenge(); |
1601 ALWAYS_INLINE bool shouldScavenge() const; | 1601 ALWAYS_INLINE bool shouldScavenge() const; |
1602 | 1602 |
1603 #if HAVE(DISPATCH_H) || OS(WINDOWS) | 1603 #if HAVE(DISPATCH_H) || OS(WINDOWS) |
(...skipping 1482 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3086 span_allocator.Init(entropy); | 3086 span_allocator.Init(entropy); |
3087 span_allocator.New(); // Reduce cache conflicts | 3087 span_allocator.New(); // Reduce cache conflicts |
3088 span_allocator.New(); // Reduce cache conflicts | 3088 span_allocator.New(); // Reduce cache conflicts |
3089 stacktrace_allocator.Init(entropy); | 3089 stacktrace_allocator.Init(entropy); |
3090 DLL_Init(&sampled_objects, entropy); | 3090 DLL_Init(&sampled_objects, entropy); |
3091 for (size_t i = 0; i < kNumClasses; ++i) { | 3091 for (size_t i = 0; i < kNumClasses; ++i) { |
3092 central_cache[i].Init(i, entropy); | 3092 central_cache[i].Init(i, entropy); |
3093 } | 3093 } |
3094 pageheap->init(); | 3094 pageheap->init(); |
3095 phinited = 1; | 3095 phinited = 1; |
3096 #if OS(DARWIN) | 3096 #if OS(MACOSX) |
3097 FastMallocZone::init(); | 3097 FastMallocZone::init(); |
3098 #endif | 3098 #endif |
3099 } | 3099 } |
3100 } | 3100 } |
3101 | 3101 |
3102 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::NewHeap(ThreadIdentifier tid,
uintptr_t entropy) { | 3102 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::NewHeap(ThreadIdentifier tid,
uintptr_t entropy) { |
3103 // Create the heap and add it to the linked list | 3103 // Create the heap and add it to the linked list |
3104 TCMalloc_ThreadCache *heap = threadheap_allocator.New(); | 3104 TCMalloc_ThreadCache *heap = threadheap_allocator.New(); |
3105 heap->Init(tid, entropy); | 3105 heap->Init(tid, entropy); |
3106 heap->next_ = thread_heaps; | 3106 heap->next_ = thread_heaps; |
(...skipping 449 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3556 const int tc_length = central_cache[cl].tc_length(); | 3556 const int tc_length = central_cache[cl].tc_length(); |
3557 | 3557 |
3558 statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length); | 3558 statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length); |
3559 } | 3559 } |
3560 for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadC
ache = threadCache->next_) | 3560 for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadC
ache = threadCache->next_) |
3561 statistics.freeListBytes += threadCache->Size(); | 3561 statistics.freeListBytes += threadCache->Size(); |
3562 | 3562 |
3563 return statistics; | 3563 return statistics; |
3564 } | 3564 } |
3565 | 3565 |
3566 #if OS(DARWIN) | 3566 #if OS(MACOSX) |
3567 | 3567 |
3568 template <typename T> | 3568 template <typename T> |
3569 T* RemoteMemoryReader::nextEntryInHardenedLinkedList(T** remoteAddress, uintptr_
t entropy) const | 3569 T* RemoteMemoryReader::nextEntryInHardenedLinkedList(T** remoteAddress, uintptr_
t entropy) const |
3570 { | 3570 { |
3571 T** localAddress = (*this)(remoteAddress); | 3571 T** localAddress = (*this)(remoteAddress); |
3572 if (!localAddress) | 3572 if (!localAddress) |
3573 return 0; | 3573 return 0; |
3574 T* hardenedNext = *localAddress; | 3574 T* hardenedNext = *localAddress; |
3575 if (!hardenedNext || hardenedNext == (void*)entropy) | 3575 if (!hardenedNext || hardenedNext == (void*)entropy) |
3576 return 0; | 3576 return 0; |
(...skipping 309 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3886 m_zone.introspect = &jscore_fastmalloc_introspection; | 3886 m_zone.introspect = &jscore_fastmalloc_introspection; |
3887 malloc_zone_register(&m_zone); | 3887 malloc_zone_register(&m_zone); |
3888 } | 3888 } |
3889 | 3889 |
3890 | 3890 |
3891 void FastMallocZone::init() | 3891 void FastMallocZone::init() |
3892 { | 3892 { |
3893 static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Cen
tral_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); | 3893 static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Cen
tral_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); |
3894 } | 3894 } |
3895 | 3895 |
3896 #endif // OS(DARWIN) | 3896 #endif // OS(MACOSX) |
3897 | 3897 |
3898 } // namespace WTF | 3898 } // namespace WTF |
3899 | 3899 |
3900 #endif // FORCE_SYSTEM_MALLOC | 3900 #endif // FORCE_SYSTEM_MALLOC |
OLD | NEW |