| OLD | NEW |
| 1 // Copyright (c) 2005, 2007, Google Inc. | 1 // Copyright (c) 2005, 2007, Google Inc. |
| 2 // All rights reserved. | 2 // All rights reserved. |
| 3 // Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserv
ed. | 3 // Copyright (C) 2005, 2006, 2007, 2008, 2009, 2011 Apple Inc. All rights reserv
ed. |
| 4 // | 4 // |
| 5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
| 6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
| 7 // met: | 7 // met: |
| 8 // | 8 // |
| 9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
| 10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
| (...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 75 // goes from about 1100 ns to about 300 ns. | 75 // goes from about 1100 ns to about 300 ns. |
| 76 | 76 |
| 77 #include "config.h" | 77 #include "config.h" |
| 78 #include "wtf/FastMalloc.h" | 78 #include "wtf/FastMalloc.h" |
| 79 | 79 |
| 80 #include "wtf/Assertions.h" | 80 #include "wtf/Assertions.h" |
| 81 #include "wtf/CPU.h" | 81 #include "wtf/CPU.h" |
| 82 #include "wtf/StdLibExtras.h" | 82 #include "wtf/StdLibExtras.h" |
| 83 #include "wtf/UnusedParam.h" | 83 #include "wtf/UnusedParam.h" |
| 84 | 84 |
| 85 #if OS(DARWIN) | 85 #if OS(MACOSX) |
| 86 #include <AvailabilityMacros.h> | 86 #include <AvailabilityMacros.h> |
| 87 #endif | 87 #endif |
| 88 | 88 |
| 89 #include <limits> | 89 #include <limits> |
| 90 #if OS(WINDOWS) | 90 #if OS(WINDOWS) |
| 91 #include <windows.h> | 91 #include <windows.h> |
| 92 #else | 92 #else |
| 93 #include <pthread.h> | 93 #include <pthread.h> |
| 94 #endif | 94 #endif |
| 95 #include <stdlib.h> | 95 #include <stdlib.h> |
| (...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 198 size_t len = strlen(src) + 1; | 198 size_t len = strlen(src) + 1; |
| 199 char* dup = static_cast<char*>(fastMalloc(len)); | 199 char* dup = static_cast<char*>(fastMalloc(len)); |
| 200 memcpy(dup, src, len); | 200 memcpy(dup, src, len); |
| 201 return dup; | 201 return dup; |
| 202 } | 202 } |
| 203 | 203 |
| 204 } // namespace WTF | 204 } // namespace WTF |
| 205 | 205 |
| 206 #if FORCE_SYSTEM_MALLOC | 206 #if FORCE_SYSTEM_MALLOC |
| 207 | 207 |
| 208 #if OS(DARWIN) | 208 #if OS(MACOSX) |
| 209 #include <malloc/malloc.h> | 209 #include <malloc/malloc.h> |
| 210 #elif OS(WINDOWS) | 210 #elif OS(WINDOWS) |
| 211 #include <malloc.h> | 211 #include <malloc.h> |
| 212 #endif | 212 #endif |
| 213 | 213 |
| 214 namespace WTF { | 214 namespace WTF { |
| 215 | 215 |
| 216 size_t fastMallocGoodSize(size_t bytes) | 216 size_t fastMallocGoodSize(size_t bytes) |
| 217 { | 217 { |
| 218 #if OS(DARWIN) | 218 #if OS(MACOSX) |
| 219 return malloc_good_size(bytes); | 219 return malloc_good_size(bytes); |
| 220 #else | 220 #else |
| 221 return bytes; | 221 return bytes; |
| 222 #endif | 222 #endif |
| 223 } | 223 } |
| 224 | 224 |
| 225 void* fastMalloc(size_t n) | 225 void* fastMalloc(size_t n) |
| 226 { | 226 { |
| 227 ASSERT(!isForbidden()); | 227 ASSERT(!isForbidden()); |
| 228 | 228 |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 262 void releaseFastMallocFreeMemory() { } | 262 void releaseFastMallocFreeMemory() { } |
| 263 | 263 |
| 264 FastMallocStatistics fastMallocStatistics() | 264 FastMallocStatistics fastMallocStatistics() |
| 265 { | 265 { |
| 266 FastMallocStatistics statistics = { 0, 0, 0 }; | 266 FastMallocStatistics statistics = { 0, 0, 0 }; |
| 267 return statistics; | 267 return statistics; |
| 268 } | 268 } |
| 269 | 269 |
| 270 } // namespace WTF | 270 } // namespace WTF |
| 271 | 271 |
| 272 #if OS(DARWIN) | 272 #if OS(MACOSX) |
| 273 // This symbol is present in the JavaScriptCore exports file even when FastMallo
c is disabled. | 273 // This symbol is present in the JavaScriptCore exports file even when FastMallo
c is disabled. |
| 274 // It will never be used in this case, so it's type and value are less interesti
ng than its presence. | 274 // It will never be used in this case, so it's type and value are less interesti
ng than its presence. |
| 275 extern "C" const int jscore_fastmalloc_introspection = 0; | 275 extern "C" const int jscore_fastmalloc_introspection = 0; |
| 276 #endif | 276 #endif |
| 277 | 277 |
| 278 #else // FORCE_SYSTEM_MALLOC | 278 #else // FORCE_SYSTEM_MALLOC |
| 279 | 279 |
| 280 #include "Compiler.h" | 280 #include "Compiler.h" |
| 281 #include "TCPackedCache.h" | 281 #include "TCPackedCache.h" |
| 282 #include "TCPageMap.h" | 282 #include "TCPageMap.h" |
| 283 #include "TCSpinLock.h" | 283 #include "TCSpinLock.h" |
| 284 #include "TCSystemAlloc.h" | 284 #include "TCSystemAlloc.h" |
| 285 #include <algorithm> | 285 #include <algorithm> |
| 286 #include <errno.h> | 286 #include <errno.h> |
| 287 #include <pthread.h> | 287 #include <pthread.h> |
| 288 #include <stdarg.h> | 288 #include <stdarg.h> |
| 289 #include <stddef.h> | 289 #include <stddef.h> |
| 290 #if OS(UNIX) | 290 #if OS(UNIX) |
| 291 #include <unistd.h> | 291 #include <unistd.h> |
| 292 #endif | 292 #endif |
| 293 #if OS(WINDOWS) | 293 #if OS(WINDOWS) |
| 294 #ifndef WIN32_LEAN_AND_MEAN | 294 #ifndef WIN32_LEAN_AND_MEAN |
| 295 #define WIN32_LEAN_AND_MEAN | 295 #define WIN32_LEAN_AND_MEAN |
| 296 #endif | 296 #endif |
| 297 #include <windows.h> | 297 #include <windows.h> |
| 298 #endif | 298 #endif |
| 299 | 299 |
| 300 #if OS(DARWIN) | 300 #if OS(MACOSX) |
| 301 #include "MallocZoneSupport.h" | 301 #include "MallocZoneSupport.h" |
| 302 #include "wtf/HashSet.h" | 302 #include "wtf/HashSet.h" |
| 303 #include "wtf/Vector.h" | 303 #include "wtf/Vector.h" |
| 304 #else | 304 #else |
| 305 #include "wtf/CurrentTime.h" | 305 #include "wtf/CurrentTime.h" |
| 306 #endif | 306 #endif |
| 307 | 307 |
| 308 #if HAVE(DISPATCH_H) | 308 #if HAVE(DISPATCH_H) |
| 309 #include <dispatch/dispatch.h> | 309 #include <dispatch/dispatch.h> |
| 310 #endif | 310 #endif |
| 311 | 311 |
| 312 #ifdef __has_include | 312 #ifdef __has_include |
| 313 #if __has_include(<System/pthread_machdep.h>) | 313 #if __has_include(<System/pthread_machdep.h>) |
| 314 | 314 |
| 315 #include <System/pthread_machdep.h> | 315 #include <System/pthread_machdep.h> |
| 316 | 316 |
| 317 #endif | 317 #endif |
| 318 #endif | 318 #endif |
| 319 | 319 |
| 320 #ifndef PRIuS | 320 #ifndef PRIuS |
| 321 #define PRIuS "zu" | 321 #define PRIuS "zu" |
| 322 #endif | 322 #endif |
| 323 | 323 |
| 324 // Calling pthread_getspecific through a global function pointer is faster than
a normal | 324 // Calling pthread_getspecific through a global function pointer is faster than
a normal |
| 325 // call to the function on Mac OS X, and it's used in performance-critical code.
So we | 325 // call to the function on Mac OS X, and it's used in performance-critical code.
So we |
| 326 // use a function pointer. But that's not necessarily faster on other platforms,
and we had | 326 // use a function pointer. But that's not necessarily faster on other platforms,
and we had |
| 327 // problems with this technique on Windows, so we'll do this only on Mac OS X. | 327 // problems with this technique on Windows, so we'll do this only on Mac OS X. |
| 328 #if OS(DARWIN) | 328 #if OS(MACOSX) |
| 329 #if !USE(PTHREAD_GETSPECIFIC_DIRECT) | 329 #if !USE(PTHREAD_GETSPECIFIC_DIRECT) |
| 330 static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_ge
tspecific; | 330 static void* (*pthread_getspecific_function_pointer)(pthread_key_t) = pthread_ge
tspecific; |
| 331 #define pthread_getspecific(key) pthread_getspecific_function_pointer(key) | 331 #define pthread_getspecific(key) pthread_getspecific_function_pointer(key) |
| 332 #else | 332 #else |
| 333 #define pthread_getspecific(key) _pthread_getspecific_direct(key) | 333 #define pthread_getspecific(key) _pthread_getspecific_direct(key) |
| 334 #define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val)) | 334 #define pthread_setspecific(key, val) _pthread_setspecific_direct(key, (val)) |
| 335 #endif | 335 #endif |
| 336 #endif | 336 #endif |
| 337 | 337 |
| 338 #define DEFINE_VARIABLE(type, name, value, meaning) \ | 338 #define DEFINE_VARIABLE(type, name, value, meaning) \ |
| (...skipping 17 matching lines...) Expand all Loading... |
| 356 #define realloc fastRealloc | 356 #define realloc fastRealloc |
| 357 | 357 |
| 358 #define MESSAGE LOG_ERROR | 358 #define MESSAGE LOG_ERROR |
| 359 #define CHECK_CONDITION ASSERT | 359 #define CHECK_CONDITION ASSERT |
| 360 | 360 |
| 361 static const char kLLHardeningMask = 0; | 361 static const char kLLHardeningMask = 0; |
| 362 template <unsigned> struct EntropySource; | 362 template <unsigned> struct EntropySource; |
| 363 template <> struct EntropySource<4> { | 363 template <> struct EntropySource<4> { |
| 364 static uint32_t value() | 364 static uint32_t value() |
| 365 { | 365 { |
| 366 #if OS(DARWIN) | 366 #if OS(MACOSX) |
| 367 return arc4random(); | 367 return arc4random(); |
| 368 #else | 368 #else |
| 369 return static_cast<uint32_t>(static_cast<uintptr_t>(currentTime() * 1000
0) ^ reinterpret_cast<uintptr_t>(&kLLHardeningMask)); | 369 return static_cast<uint32_t>(static_cast<uintptr_t>(currentTime() * 1000
0) ^ reinterpret_cast<uintptr_t>(&kLLHardeningMask)); |
| 370 #endif | 370 #endif |
| 371 } | 371 } |
| 372 }; | 372 }; |
| 373 | 373 |
| 374 template <> struct EntropySource<8> { | 374 template <> struct EntropySource<8> { |
| 375 static uint64_t value() | 375 static uint64_t value() |
| 376 { | 376 { |
| (...skipping 554 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 931 | 931 |
| 932 void Delete(T* p) { | 932 void Delete(T* p) { |
| 933 HardenedSLL new_head = HardenedSLL::create(p); | 933 HardenedSLL new_head = HardenedSLL::create(p); |
| 934 SLL_SetNext(new_head, free_list_, entropy_); | 934 SLL_SetNext(new_head, free_list_, entropy_); |
| 935 free_list_ = new_head; | 935 free_list_ = new_head; |
| 936 inuse_--; | 936 inuse_--; |
| 937 } | 937 } |
| 938 | 938 |
| 939 int inuse() const { return inuse_; } | 939 int inuse() const { return inuse_; } |
| 940 | 940 |
| 941 #if OS(DARWIN) | 941 #if OS(MACOSX) |
| 942 template <class Recorder> | 942 template <class Recorder> |
| 943 void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader&
reader) | 943 void recordAdministrativeRegions(Recorder& recorder, const RemoteMemoryReader&
reader) |
| 944 { | 944 { |
| 945 for (HardenedSLL adminAllocation = allocated_regions_; adminAllocation; ad
minAllocation.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<voi
d**>(adminAllocation.value()), entropy_))) | 945 for (HardenedSLL adminAllocation = allocated_regions_; adminAllocation; ad
minAllocation.setValue(reader.nextEntryInHardenedLinkedList(reinterpret_cast<voi
d**>(adminAllocation.value()), entropy_))) |
| 946 recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation.v
alue()), kAllocIncrement); | 946 recorder.recordRegion(reinterpret_cast<vm_address_t>(adminAllocation.v
alue()), kAllocIncrement); |
| 947 } | 947 } |
| 948 #endif | 948 #endif |
| 949 }; | 949 }; |
| 950 | 950 |
| 951 // ------------------------------------------------------------------------- | 951 // ------------------------------------------------------------------------- |
| (...skipping 313 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1265 // to be exactly 64 bytes. | 1265 // to be exactly 64 bytes. |
| 1266 template <> class TCMalloc_Central_FreeListPadded_Template<0> : public TCMalloc_
Central_FreeList { | 1266 template <> class TCMalloc_Central_FreeListPadded_Template<0> : public TCMalloc_
Central_FreeList { |
| 1267 }; | 1267 }; |
| 1268 | 1268 |
| 1269 typedef TCMalloc_Central_FreeListPadded_Template<sizeof(TCMalloc_Central_FreeLis
t) % 64> TCMalloc_Central_FreeListPadded; | 1269 typedef TCMalloc_Central_FreeListPadded_Template<sizeof(TCMalloc_Central_FreeLis
t) % 64> TCMalloc_Central_FreeListPadded; |
| 1270 | 1270 |
| 1271 #if COMPILER(CLANG) && defined(__has_warning) | 1271 #if COMPILER(CLANG) && defined(__has_warning) |
| 1272 #pragma clang diagnostic pop | 1272 #pragma clang diagnostic pop |
| 1273 #endif | 1273 #endif |
| 1274 | 1274 |
| 1275 #if OS(DARWIN) | 1275 #if OS(MACOSX) |
| 1276 struct Span; | 1276 struct Span; |
| 1277 class TCMalloc_PageHeap; | 1277 class TCMalloc_PageHeap; |
| 1278 class TCMalloc_ThreadCache; | 1278 class TCMalloc_ThreadCache; |
| 1279 template <typename T> class PageHeapAllocator; | 1279 template <typename T> class PageHeapAllocator; |
| 1280 | 1280 |
| 1281 class FastMallocZone { | 1281 class FastMallocZone { |
| 1282 public: | 1282 public: |
| 1283 static void init(); | 1283 static void init(); |
| 1284 | 1284 |
| 1285 static kern_return_t enumerate(task_t, void*, unsigned typeMmask, vm_address
_t zoneAddress, memory_reader_t, vm_range_recorder_t); | 1285 static kern_return_t enumerate(task_t, void*, unsigned typeMmask, vm_address
_t zoneAddress, memory_reader_t, vm_range_recorder_t); |
| (...skipping 297 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1583 // IncrementalScavenge(n) is called whenever n pages are freed. | 1583 // IncrementalScavenge(n) is called whenever n pages are freed. |
| 1584 void IncrementalScavenge(Length n); | 1584 void IncrementalScavenge(Length n); |
| 1585 #endif | 1585 #endif |
| 1586 | 1586 |
| 1587 // Number of pages to deallocate before doing more scavenging | 1587 // Number of pages to deallocate before doing more scavenging |
| 1588 int64_t scavenge_counter_; | 1588 int64_t scavenge_counter_; |
| 1589 | 1589 |
| 1590 // Index of last free list we scavenged | 1590 // Index of last free list we scavenged |
| 1591 size_t scavenge_index_; | 1591 size_t scavenge_index_; |
| 1592 | 1592 |
| 1593 #if OS(DARWIN) | 1593 #if OS(MACOSX) |
| 1594 friend class FastMallocZone; | 1594 friend class FastMallocZone; |
| 1595 #endif | 1595 #endif |
| 1596 | 1596 |
| 1597 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY | 1597 #if USE_BACKGROUND_THREAD_TO_SCAVENGE_MEMORY |
| 1598 void initializeScavenger(); | 1598 void initializeScavenger(); |
| 1599 ALWAYS_INLINE void signalScavenger(); | 1599 ALWAYS_INLINE void signalScavenger(); |
| 1600 void scavenge(); | 1600 void scavenge(); |
| 1601 ALWAYS_INLINE bool shouldScavenge() const; | 1601 ALWAYS_INLINE bool shouldScavenge() const; |
| 1602 | 1602 |
| 1603 #if HAVE(DISPATCH_H) || OS(WINDOWS) | 1603 #if HAVE(DISPATCH_H) || OS(WINDOWS) |
| (...skipping 1482 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3086 span_allocator.Init(entropy); | 3086 span_allocator.Init(entropy); |
| 3087 span_allocator.New(); // Reduce cache conflicts | 3087 span_allocator.New(); // Reduce cache conflicts |
| 3088 span_allocator.New(); // Reduce cache conflicts | 3088 span_allocator.New(); // Reduce cache conflicts |
| 3089 stacktrace_allocator.Init(entropy); | 3089 stacktrace_allocator.Init(entropy); |
| 3090 DLL_Init(&sampled_objects, entropy); | 3090 DLL_Init(&sampled_objects, entropy); |
| 3091 for (size_t i = 0; i < kNumClasses; ++i) { | 3091 for (size_t i = 0; i < kNumClasses; ++i) { |
| 3092 central_cache[i].Init(i, entropy); | 3092 central_cache[i].Init(i, entropy); |
| 3093 } | 3093 } |
| 3094 pageheap->init(); | 3094 pageheap->init(); |
| 3095 phinited = 1; | 3095 phinited = 1; |
| 3096 #if OS(DARWIN) | 3096 #if OS(MACOSX) |
| 3097 FastMallocZone::init(); | 3097 FastMallocZone::init(); |
| 3098 #endif | 3098 #endif |
| 3099 } | 3099 } |
| 3100 } | 3100 } |
| 3101 | 3101 |
| 3102 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::NewHeap(ThreadIdentifier tid,
uintptr_t entropy) { | 3102 inline TCMalloc_ThreadCache* TCMalloc_ThreadCache::NewHeap(ThreadIdentifier tid,
uintptr_t entropy) { |
| 3103 // Create the heap and add it to the linked list | 3103 // Create the heap and add it to the linked list |
| 3104 TCMalloc_ThreadCache *heap = threadheap_allocator.New(); | 3104 TCMalloc_ThreadCache *heap = threadheap_allocator.New(); |
| 3105 heap->Init(tid, entropy); | 3105 heap->Init(tid, entropy); |
| 3106 heap->next_ = thread_heaps; | 3106 heap->next_ = thread_heaps; |
| (...skipping 449 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3556 const int tc_length = central_cache[cl].tc_length(); | 3556 const int tc_length = central_cache[cl].tc_length(); |
| 3557 | 3557 |
| 3558 statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length); | 3558 statistics.freeListBytes += ByteSizeForClass(cl) * (length + tc_length); |
| 3559 } | 3559 } |
| 3560 for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadC
ache = threadCache->next_) | 3560 for (TCMalloc_ThreadCache* threadCache = thread_heaps; threadCache ; threadC
ache = threadCache->next_) |
| 3561 statistics.freeListBytes += threadCache->Size(); | 3561 statistics.freeListBytes += threadCache->Size(); |
| 3562 | 3562 |
| 3563 return statistics; | 3563 return statistics; |
| 3564 } | 3564 } |
| 3565 | 3565 |
| 3566 #if OS(DARWIN) | 3566 #if OS(MACOSX) |
| 3567 | 3567 |
| 3568 template <typename T> | 3568 template <typename T> |
| 3569 T* RemoteMemoryReader::nextEntryInHardenedLinkedList(T** remoteAddress, uintptr_
t entropy) const | 3569 T* RemoteMemoryReader::nextEntryInHardenedLinkedList(T** remoteAddress, uintptr_
t entropy) const |
| 3570 { | 3570 { |
| 3571 T** localAddress = (*this)(remoteAddress); | 3571 T** localAddress = (*this)(remoteAddress); |
| 3572 if (!localAddress) | 3572 if (!localAddress) |
| 3573 return 0; | 3573 return 0; |
| 3574 T* hardenedNext = *localAddress; | 3574 T* hardenedNext = *localAddress; |
| 3575 if (!hardenedNext || hardenedNext == (void*)entropy) | 3575 if (!hardenedNext || hardenedNext == (void*)entropy) |
| 3576 return 0; | 3576 return 0; |
| (...skipping 309 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3886 m_zone.introspect = &jscore_fastmalloc_introspection; | 3886 m_zone.introspect = &jscore_fastmalloc_introspection; |
| 3887 malloc_zone_register(&m_zone); | 3887 malloc_zone_register(&m_zone); |
| 3888 } | 3888 } |
| 3889 | 3889 |
| 3890 | 3890 |
| 3891 void FastMallocZone::init() | 3891 void FastMallocZone::init() |
| 3892 { | 3892 { |
| 3893 static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Cen
tral_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); | 3893 static FastMallocZone zone(pageheap, &thread_heaps, static_cast<TCMalloc_Cen
tral_FreeListPadded*>(central_cache), &span_allocator, &threadheap_allocator); |
| 3894 } | 3894 } |
| 3895 | 3895 |
| 3896 #endif // OS(DARWIN) | 3896 #endif // OS(MACOSX) |
| 3897 | 3897 |
| 3898 } // namespace WTF | 3898 } // namespace WTF |
| 3899 | 3899 |
| 3900 #endif // FORCE_SYSTEM_MALLOC | 3900 #endif // FORCE_SYSTEM_MALLOC |
| OLD | NEW |