Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(644)

Side by Side Diff: Source/platform/heap/Heap.h

Issue 911663002: Revert of Oilpan: Rename heap classes (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 5 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | Source/platform/heap/Heap.cpp » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * Copyright (C) 2013 Google Inc. All rights reserved. 2 * Copyright (C) 2013 Google Inc. All rights reserved.
3 * 3 *
4 * Redistribution and use in source and binary forms, with or without 4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are 5 * modification, are permitted provided that the following conditions are
6 * met: 6 * met:
7 * 7 *
8 * * Redistributions of source code must retain the above copyright 8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer. 9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above 10 * * Redistributions in binary form must reproduce the above
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after
82 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) 82 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
83 #define FILL_ZERO_IF_PRODUCTION(address, size) do { } while (false) 83 #define FILL_ZERO_IF_PRODUCTION(address, size) do { } while (false)
84 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) memset((address), 0, (size)) 84 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) memset((address), 0, (size))
85 #else 85 #else
86 #define FILL_ZERO_IF_PRODUCTION(address, size) memset((address), 0, (size)) 86 #define FILL_ZERO_IF_PRODUCTION(address, size) memset((address), 0, (size))
87 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) do { } while (false) 87 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) do { } while (false)
88 #endif 88 #endif
89 89
90 class CallbackStack; 90 class CallbackStack;
91 class PageMemory; 91 class PageMemory;
92 class NormalPageHeap; 92 class ThreadHeapForHeapPage;
93 template<ThreadAffinity affinity> class ThreadLocalPersistents; 93 template<ThreadAffinity affinity> class ThreadLocalPersistents;
94 template<typename T, typename RootsAccessor = ThreadLocalPersistents<ThreadingTr ait<T>::Affinity>> class Persistent; 94 template<typename T, typename RootsAccessor = ThreadLocalPersistents<ThreadingTr ait<T>::Affinity>> class Persistent;
95 95
96 #if ENABLE(GC_PROFILING) 96 #if ENABLE(GC_PROFILING)
97 class TracedValue; 97 class TracedValue;
98 #endif 98 #endif
99 99
100 // HeapObjectHeader is 4 byte (32 bit) that has the following layout: 100 // HeapObjectHeader is 4 byte (32 bit) that has the following layout:
101 // 101 //
102 // | gcInfoIndex (15 bit) | size (14 bit) | dead bit (1 bit) | freed bit (1 bit) | mark bit (1 bit) | 102 // | gcInfoIndex (15 bit) | size (14 bit) | dead bit (1 bit) | freed bit (1 bit) | mark bit (1 bit) |
103 // 103 //
104 // - For non-large objects, 14 bit is enough for |size| because the blink 104 // - For non-large objects, 14 bit is enough for |size| because the blink
105 // page size is 2^17 byte and each object is guaranteed to be aligned with 105 // page size is 2^17 byte and each object is guaranteed to be aligned with
106 // 2^3 byte. 106 // 2^3 byte.
107 // - For large objects, |size| is 0. The actual size of a large object is 107 // - For large objects, |size| is 0. The actual size of a large object is
108 // stored in LargeObjectPage::m_payloadSize. 108 // stored in LargeObject::m_payloadSize.
109 // - 15 bit is enough for gcInfoIndex because there are less than 2^15 types 109 // - 15 bit is enough for gcInfoIndex because there are less than 2^15 types
110 // in Blink. 110 // in Blink.
111 const size_t headerGCInfoIndexShift = 17; 111 const size_t headerGCInfoIndexShift = 17;
112 const size_t headerGCInfoIndexMask = (static_cast<size_t>((1 << 15) - 1)) << hea derGCInfoIndexShift; 112 const size_t headerGCInfoIndexMask = (static_cast<size_t>((1 << 15) - 1)) << hea derGCInfoIndexShift;
113 const size_t headerSizeMask = (static_cast<size_t>((1 << 14) - 1)) << 3; 113 const size_t headerSizeMask = (static_cast<size_t>((1 << 14) - 1)) << 3;
114 const size_t headerMarkBitMask = 1; 114 const size_t headerMarkBitMask = 1;
115 const size_t headerFreedBitMask = 2; 115 const size_t headerFreedBitMask = 2;
116 // The dead bit is used for objects that have gone through a GC marking, but did 116 // The dead bit is used for objects that have gone through a GC marking, but did
117 // not get swept before a new GC started. In that case we set the dead bit on 117 // not get swept before a new GC started. In that case we set the dead bit on
118 // objects that were not marked in the previous GC to ensure we are not tracing 118 // objects that were not marked in the previous GC to ensure we are not tracing
119 // them via a conservatively found pointer. Tracing dead objects could lead to 119 // them via a conservatively found pointer. Tracing dead objects could lead to
120 // tracing of already finalized objects in another thread's heap which is a 120 // tracing of already finalized objects in another thread's heap which is a
121 // use-after-free situation. 121 // use-after-free situation.
122 const size_t headerDeadBitMask = 4; 122 const size_t headerDeadBitMask = 4;
123 // On free-list entries we reuse the dead bit to distinguish a normal free-list 123 // On free-list entries we reuse the dead bit to distinguish a normal free-list
124 // entry from one that has been promptly freed. 124 // entry from one that has been promptly freed.
125 const size_t headerPromptlyFreedBitMask = headerFreedBitMask | headerDeadBitMask ; 125 const size_t headerPromptlyFreedBitMask = headerFreedBitMask | headerDeadBitMask ;
126 const size_t largeObjectSizeInHeader = 0; 126 const size_t largeObjectSizeInHeader = 0;
127 const size_t gcInfoIndexForFreeListHeader = 0; 127 const size_t gcInfoIndexForFreeListHeader = 0;
128 const size_t nonLargeObjectPageSizeMax = 1 << 17; 128 const size_t nonLargeObjectSizeMax = 1 << 17;
129 129
130 static_assert(nonLargeObjectPageSizeMax >= blinkPageSize, "max size supported by HeapObjectHeader must at least be blinkPageSize"); 130 static_assert(nonLargeObjectSizeMax >= blinkPageSize, "max size supported by Hea pObjectHeader must at least be blinkPageSize");
131 131
132 class PLATFORM_EXPORT HeapObjectHeader { 132 class PLATFORM_EXPORT HeapObjectHeader {
133 public: 133 public:
134 // If gcInfoIndex is 0, this header is interpreted as a free list header. 134 // If gcInfoIndex is 0, this header is interpreted as a free list header.
135 NO_SANITIZE_ADDRESS 135 NO_SANITIZE_ADDRESS
136 HeapObjectHeader(size_t size, size_t gcInfoIndex) 136 HeapObjectHeader(size_t size, size_t gcInfoIndex)
137 { 137 {
138 #if ENABLE(ASSERT) 138 #if ENABLE(ASSERT)
139 m_magic = magic; 139 m_magic = magic;
140 #endif 140 #endif
141 #if ENABLE(GC_PROFILING) 141 #if ENABLE(GC_PROFILING)
142 m_age = 0; 142 m_age = 0;
143 #endif 143 #endif
144 // sizeof(HeapObjectHeader) must be equal to or smaller than 144 // sizeof(HeapObjectHeader) must be equal to or smaller than
145 // allocationGranurarity, because HeapObjectHeader is used as a header 145 // allocationGranurarity, because HeapObjectHeader is used as a header
146 // for an freed entry. Given that the smallest entry size is 146 // for an freed entry. Given that the smallest entry size is
147 // allocationGranurarity, HeapObjectHeader must fit into the size. 147 // allocationGranurarity, HeapObjectHeader must fit into the size.
148 static_assert(sizeof(HeapObjectHeader) <= allocationGranularity, "size o f HeapObjectHeader must be smaller than allocationGranularity"); 148 static_assert(sizeof(HeapObjectHeader) <= allocationGranularity, "size o f HeapObjectHeader must be smaller than allocationGranularity");
149 #if CPU(64BIT) 149 #if CPU(64BIT)
150 static_assert(sizeof(HeapObjectHeader) == 8, "size of HeapObjectHeader m ust be 8 byte aligned"); 150 static_assert(sizeof(HeapObjectHeader) == 8, "size of HeapObjectHeader m ust be 8 byte aligned");
151 #endif 151 #endif
152 152
153 ASSERT(gcInfoIndex < GCInfoTable::maxIndex); 153 ASSERT(gcInfoIndex < GCInfoTable::maxIndex);
154 ASSERT(size < nonLargeObjectPageSizeMax); 154 ASSERT(size < nonLargeObjectSizeMax);
155 ASSERT(!(size & allocationMask)); 155 ASSERT(!(size & allocationMask));
156 m_encoded = (gcInfoIndex << headerGCInfoIndexShift) | size | (gcInfoInde x ? 0 : headerFreedBitMask); 156 m_encoded = (gcInfoIndex << headerGCInfoIndexShift) | size | (gcInfoInde x ? 0 : headerFreedBitMask);
157 } 157 }
158 158
159 NO_SANITIZE_ADDRESS 159 NO_SANITIZE_ADDRESS
160 bool isFree() const { return m_encoded & headerFreedBitMask; } 160 bool isFree() const { return m_encoded & headerFreedBitMask; }
161 NO_SANITIZE_ADDRESS 161 NO_SANITIZE_ADDRESS
162 bool isPromptlyFreed() const { return (m_encoded & headerPromptlyFreedBitMas k) == headerPromptlyFreedBitMask; } 162 bool isPromptlyFreed() const { return (m_encoded & headerPromptlyFreedBitMas k) == headerPromptlyFreedBitMask; }
163 NO_SANITIZE_ADDRESS 163 NO_SANITIZE_ADDRESS
164 void markPromptlyFreed() { m_encoded |= headerPromptlyFreedBitMask; } 164 void markPromptlyFreed() { m_encoded |= headerPromptlyFreedBitMask; }
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after
350 // header should be OS page size away from being Blink page size 350 // header should be OS page size away from being Blink page size
351 // aligned. 351 // aligned.
352 inline bool isPageHeaderAddress(Address address) 352 inline bool isPageHeaderAddress(Address address)
353 { 353 {
354 return !((reinterpret_cast<uintptr_t>(address) & blinkPageOffsetMask) - WTF: :kSystemPageSize); 354 return !((reinterpret_cast<uintptr_t>(address) & blinkPageOffsetMask) - WTF: :kSystemPageSize);
355 } 355 }
356 #endif 356 #endif
357 357
358 // FIXME: Add a good comment about the heap layout once heap relayout work 358 // FIXME: Add a good comment about the heap layout once heap relayout work
359 // is done. 359 // is done.
360 class BasePage { 360 class BaseHeapPage {
361 public: 361 public:
362 BasePage(PageMemory*, BaseHeap*); 362 BaseHeapPage(PageMemory*, ThreadHeap*);
363 virtual ~BasePage() { } 363 virtual ~BaseHeapPage() { }
364 364
365 void link(BasePage** previousNext) 365 void link(BaseHeapPage** previousNext)
366 { 366 {
367 m_next = *previousNext; 367 m_next = *previousNext;
368 *previousNext = this; 368 *previousNext = this;
369 } 369 }
370 void unlink(BasePage** previousNext) 370 void unlink(BaseHeapPage** previousNext)
371 { 371 {
372 *previousNext = m_next; 372 *previousNext = m_next;
373 m_next = nullptr; 373 m_next = nullptr;
374 } 374 }
375 BasePage* next() const { return m_next; } 375 BaseHeapPage* next() const { return m_next; }
376 376
377 // virtual methods are slow. So performance-sensitive methods 377 // virtual methods are slow. So performance-sensitive methods
378 // should be defined as non-virtual methods on NormalPage and LargeObjectPag e. 378 // should be defined as non-virtual methods on HeapPage and LargeObject.
379 // The following methods are not performance-sensitive. 379 // The following methods are not performance-sensitive.
380 virtual size_t objectPayloadSizeForTesting() = 0; 380 virtual size_t objectPayloadSizeForTesting() = 0;
381 virtual bool isEmpty() = 0; 381 virtual bool isEmpty() = 0;
382 virtual void removeFromHeap() = 0; 382 virtual void removeFromHeap() = 0;
383 virtual void sweep() = 0; 383 virtual void sweep() = 0;
384 virtual void markUnmarkedObjectsDead() = 0; 384 virtual void markUnmarkedObjectsDead() = 0;
385 // Check if the given address points to an object in this 385 // Check if the given address points to an object in this
386 // heap page. If so, find the start of that object and mark it 386 // heap page. If so, find the start of that object and mark it
387 // using the given Visitor. Otherwise do nothing. The pointer must 387 // using the given Visitor. Otherwise do nothing. The pointer must
388 // be within the same aligned blinkPageSize as the this-pointer. 388 // be within the same aligned blinkPageSize as the this-pointer.
389 // 389 //
390 // This is used during conservative stack scanning to 390 // This is used during conservative stack scanning to
391 // conservatively mark all objects that could be referenced from 391 // conservatively mark all objects that could be referenced from
392 // the stack. 392 // the stack.
393 virtual void checkAndMarkPointer(Visitor*, Address) = 0; 393 virtual void checkAndMarkPointer(Visitor*, Address) = 0;
394 virtual void markOrphaned(); 394 virtual void markOrphaned();
395 #if ENABLE(GC_PROFILING) 395 #if ENABLE(GC_PROFILING)
396 virtual const GCInfo* findGCInfo(Address) = 0; 396 virtual const GCInfo* findGCInfo(Address) = 0;
397 virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*) = 0; 397 virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*) = 0;
398 #endif 398 #endif
399 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING) 399 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
400 virtual bool contains(Address) = 0; 400 virtual bool contains(Address) = 0;
401 #endif 401 #endif
402 virtual size_t size() = 0; 402 virtual size_t size() = 0;
403 virtual bool isLargeObjectPage() { return false; } 403 virtual bool isLargeObject() { return false; }
404 404
405 Address address() { return reinterpret_cast<Address>(this); } 405 Address address() { return reinterpret_cast<Address>(this); }
406 PageMemory* storage() const { return m_storage; } 406 PageMemory* storage() const { return m_storage; }
407 BaseHeap* heap() const { return m_heap; } 407 ThreadHeap* heap() const { return m_heap; }
408 bool orphaned() { return !m_heap; } 408 bool orphaned() { return !m_heap; }
409 bool terminating() { return m_terminating; } 409 bool terminating() { return m_terminating; }
410 void setTerminating() { m_terminating = true; } 410 void setTerminating() { m_terminating = true; }
411 411
412 // Returns true if this page has been swept by the ongoing lazy sweep. 412 // Returns true if this page has been swept by the ongoing lazy sweep.
413 bool hasBeenSwept() const { return m_swept; } 413 bool hasBeenSwept() const { return m_swept; }
414 414
415 void markAsSwept() 415 void markAsSwept()
416 { 416 {
417 ASSERT(!m_swept); 417 ASSERT(!m_swept);
418 m_swept = true; 418 m_swept = true;
419 } 419 }
420 420
421 void markAsUnswept() 421 void markAsUnswept()
422 { 422 {
423 ASSERT(m_swept); 423 ASSERT(m_swept);
424 m_swept = false; 424 m_swept = false;
425 } 425 }
426 426
427 private: 427 private:
428 PageMemory* m_storage; 428 PageMemory* m_storage;
429 BaseHeap* m_heap; 429 ThreadHeap* m_heap;
430 BasePage* m_next; 430 BaseHeapPage* m_next;
431 // Whether the page is part of a terminating thread or not. 431 // Whether the page is part of a terminating thread or not.
432 bool m_terminating; 432 bool m_terminating;
433 433
434 // Track the sweeping state of a page. Set to true once 434 // Track the sweeping state of a page. Set to true once
435 // the lazy sweep completes has processed it. 435 // the lazy sweep completes has processed it.
436 // 436 //
437 // Set to false at the start of a sweep, true upon completion 437 // Set to false at the start of a sweep, true upon completion
438 // of lazy sweeping. 438 // of lazy sweeping.
439 bool m_swept; 439 bool m_swept;
440 friend class BaseHeap; 440 friend class ThreadHeap;
441 }; 441 };
442 442
443 class NormalPage final : public BasePage { 443 class HeapPage final : public BaseHeapPage {
444 public: 444 public:
445 NormalPage(PageMemory*, BaseHeap*); 445 HeapPage(PageMemory*, ThreadHeap*);
446 446
447 Address payload() 447 Address payload()
448 { 448 {
449 return address() + sizeof(NormalPage) + headerPadding(); 449 return address() + sizeof(HeapPage) + headerPadding();
450 } 450 }
451 size_t payloadSize() 451 size_t payloadSize()
452 { 452 {
453 return (blinkPagePayloadSize() - sizeof(NormalPage) - headerPadding()) & ~allocationMask; 453 return (blinkPagePayloadSize() - sizeof(HeapPage) - headerPadding()) & ~ allocationMask;
454 } 454 }
455 Address payloadEnd() { return payload() + payloadSize(); } 455 Address payloadEnd() { return payload() + payloadSize(); }
456 bool containedInObjectPayload(Address address) { return payload() <= address && address < payloadEnd(); } 456 bool containedInObjectPayload(Address address) { return payload() <= address && address < payloadEnd(); }
457 457
458 virtual size_t objectPayloadSizeForTesting() override; 458 virtual size_t objectPayloadSizeForTesting() override;
459 virtual bool isEmpty() override; 459 virtual bool isEmpty() override;
460 virtual void removeFromHeap() override; 460 virtual void removeFromHeap() override;
461 virtual void sweep() override; 461 virtual void sweep() override;
462 virtual void markUnmarkedObjectsDead() override; 462 virtual void markUnmarkedObjectsDead() override;
463 virtual void checkAndMarkPointer(Visitor*, Address) override; 463 virtual void checkAndMarkPointer(Visitor*, Address) override;
464 virtual void markOrphaned() override 464 virtual void markOrphaned() override
465 { 465 {
466 // Zap the payload with a recognizable value to detect any incorrect 466 // Zap the payload with a recognizable value to detect any incorrect
467 // cross thread pointer usage. 467 // cross thread pointer usage.
468 #if defined(ADDRESS_SANITIZER) 468 #if defined(ADDRESS_SANITIZER)
469 // This needs to zap poisoned memory as well. 469 // This needs to zap poisoned memory as well.
470 // Force unpoison memory before memset. 470 // Force unpoison memory before memset.
471 ASAN_UNPOISON_MEMORY_REGION(payload(), payloadSize()); 471 ASAN_UNPOISON_MEMORY_REGION(payload(), payloadSize());
472 #endif 472 #endif
473 memset(payload(), orphanedZapValue, payloadSize()); 473 memset(payload(), orphanedZapValue, payloadSize());
474 BasePage::markOrphaned(); 474 BaseHeapPage::markOrphaned();
475 } 475 }
476 #if ENABLE(GC_PROFILING) 476 #if ENABLE(GC_PROFILING)
477 const GCInfo* findGCInfo(Address) override; 477 const GCInfo* findGCInfo(Address) override;
478 virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*); 478 virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*);
479 void incrementMarkedObjectsAge(); 479 void incrementMarkedObjectsAge();
480 void countMarkedObjects(ClassAgeCountsMap&); 480 void countMarkedObjects(ClassAgeCountsMap&);
481 void countObjectsToSweep(ClassAgeCountsMap&); 481 void countObjectsToSweep(ClassAgeCountsMap&);
482 #endif 482 #endif
483 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING) 483 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
484 // Returns true for the whole blinkPageSize page that the page is on, even 484 // Returns true for the whole blinkPageSize page that the page is on, even
485 // for the header, and the unmapped guard page at the start. That ensures 485 // for the header, and the unmapped guard page at the start. That ensures
486 // the result can be used to populate the negative page cache. 486 // the result can be used to populate the negative page cache.
487 virtual bool contains(Address addr) override 487 virtual bool contains(Address addr) override
488 { 488 {
489 Address blinkPageStart = roundToBlinkPageStart(address()); 489 Address blinkPageStart = roundToBlinkPageStart(address());
490 ASSERT(blinkPageStart == address() - WTF::kSystemPageSize); // Page is a t aligned address plus guard page size. 490 ASSERT(blinkPageStart == address() - WTF::kSystemPageSize); // Page is a t aligned address plus guard page size.
491 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize; 491 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize;
492 } 492 }
493 #endif 493 #endif
494 virtual size_t size() override { return blinkPageSize; } 494 virtual size_t size() override { return blinkPageSize; }
495 495
496 NormalPageHeap* heapForNormalPage(); 496 ThreadHeapForHeapPage* heapForHeapPage();
497 void clearObjectStartBitMap(); 497 void clearObjectStartBitMap();
498 498
499 #if defined(ADDRESS_SANITIZER) 499 #if defined(ADDRESS_SANITIZER)
500 void poisonUnmarkedObjects(); 500 void poisonUnmarkedObjects();
501 #endif 501 #endif
502 502
503 // This method is needed just to avoid compilers from removing m_padding. 503 // This method is needed just to avoid compilers from removing m_padding.
504 uint64_t unusedMethod() const { return m_padding; } 504 uint64_t unusedMethod() const { return m_padding; }
505 505
506 private: 506 private:
507 HeapObjectHeader* findHeaderFromAddress(Address); 507 HeapObjectHeader* findHeaderFromAddress(Address);
508 void populateObjectStartBitMap(); 508 void populateObjectStartBitMap();
509 bool isObjectStartBitMapComputed() { return m_objectStartBitMapComputed; } 509 bool isObjectStartBitMapComputed() { return m_objectStartBitMapComputed; }
510 510
511 bool m_objectStartBitMapComputed; 511 bool m_objectStartBitMapComputed;
512 uint8_t m_objectStartBitMap[reservedForObjectBitMap]; 512 uint8_t m_objectStartBitMap[reservedForObjectBitMap];
513 uint64_t m_padding; // Preserve 8-byte alignment on 32-bit systems. 513 uint64_t m_padding; // Preserve 8-byte alignment on 32-bit systems.
514 }; 514 };
515 515
516 // Large allocations are allocated as separate objects and linked in a list. 516 // Large allocations are allocated as separate objects and linked in a list.
517 // 517 //
518 // In order to use the same memory allocation routines for everything allocated 518 // In order to use the same memory allocation routines for everything allocated
519 // in the heap, large objects are considered heap pages containing only one 519 // in the heap, large objects are considered heap pages containing only one
520 // object. 520 // object.
521 class LargeObjectPage final : public BasePage { 521 class LargeObject final : public BaseHeapPage {
522 public: 522 public:
523 LargeObjectPage(PageMemory* storage, BaseHeap* heap, size_t payloadSize) 523 LargeObject(PageMemory* storage, ThreadHeap* heap, size_t payloadSize)
524 : BasePage(storage, heap) 524 : BaseHeapPage(storage, heap)
525 , m_payloadSize(payloadSize) 525 , m_payloadSize(payloadSize)
526 { 526 {
527 } 527 }
528 528
529 Address payload() { return heapObjectHeader()->payload(); } 529 Address payload() { return heapObjectHeader()->payload(); }
530 size_t payloadSize() { return m_payloadSize; } 530 size_t payloadSize() { return m_payloadSize; }
531 Address payloadEnd() { return payload() + payloadSize(); } 531 Address payloadEnd() { return payload() + payloadSize(); }
532 bool containedInObjectPayload(Address address) { return payload() <= address && address < payloadEnd(); } 532 bool containedInObjectPayload(Address address) { return payload() <= address && address < payloadEnd(); }
533 533
534 virtual size_t objectPayloadSizeForTesting() override; 534 virtual size_t objectPayloadSizeForTesting() override;
535 virtual bool isEmpty() override; 535 virtual bool isEmpty() override;
536 virtual void removeFromHeap() override; 536 virtual void removeFromHeap() override;
537 virtual void sweep() override; 537 virtual void sweep() override;
538 virtual void markUnmarkedObjectsDead() override; 538 virtual void markUnmarkedObjectsDead() override;
539 virtual void checkAndMarkPointer(Visitor*, Address) override; 539 virtual void checkAndMarkPointer(Visitor*, Address) override;
540 virtual void markOrphaned() override 540 virtual void markOrphaned() override
541 { 541 {
542 // Zap the payload with a recognizable value to detect any incorrect 542 // Zap the payload with a recognizable value to detect any incorrect
543 // cross thread pointer usage. 543 // cross thread pointer usage.
544 memset(payload(), orphanedZapValue, payloadSize()); 544 memset(payload(), orphanedZapValue, payloadSize());
545 BasePage::markOrphaned(); 545 BaseHeapPage::markOrphaned();
546 } 546 }
547 547
548 #if ENABLE(GC_PROFILING) 548 #if ENABLE(GC_PROFILING)
549 virtual const GCInfo* findGCInfo(Address) override; 549 virtual const GCInfo* findGCInfo(Address) override;
550 virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*) override; 550 virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*) override;
551 void incrementMarkedObjectsAge(); 551 void incrementMarkedObjectsAge();
552 void countMarkedObjects(ClassAgeCountsMap&); 552 void countMarkedObjects(ClassAgeCountsMap&);
553 void countObjectsToSweep(ClassAgeCountsMap&); 553 void countObjectsToSweep(ClassAgeCountsMap&);
554 #endif 554 #endif
555 555
556 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING) 556 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
557 // Returns true for any address that is on one of the pages that this 557 // Returns true for any address that is on one of the pages that this
558 // large object uses. That ensures that we can use a negative result to 558 // large object uses. That ensures that we can use a negative result to
559 // populate the negative page cache. 559 // populate the negative page cache.
560 virtual bool contains(Address object) override 560 virtual bool contains(Address object) override
561 { 561 {
562 return roundToBlinkPageStart(address()) <= object && object < roundToBli nkPageEnd(address() + size()); 562 return roundToBlinkPageStart(address()) <= object && object < roundToBli nkPageEnd(address() + size());
563 } 563 }
564 #endif 564 #endif
565 virtual size_t size() 565 virtual size_t size()
566 { 566 {
567 return sizeof(LargeObjectPage) + headerPadding() + sizeof(HeapObjectHea der) + m_payloadSize; 567 return sizeof(LargeObject) + headerPadding() + sizeof(HeapObjectHeader) + m_payloadSize;
568 } 568 }
569 virtual bool isLargeObjectPage() override { return true; } 569 virtual bool isLargeObject() override { return true; }
570 570
571 HeapObjectHeader* heapObjectHeader() 571 HeapObjectHeader* heapObjectHeader()
572 { 572 {
573 Address headerAddress = address() + sizeof(LargeObjectPage) + headerPadd ing(); 573 Address headerAddress = address() + sizeof(LargeObject) + headerPadding( );
574 return reinterpret_cast<HeapObjectHeader*>(headerAddress); 574 return reinterpret_cast<HeapObjectHeader*>(headerAddress);
575 } 575 }
576 576
577 // This method is needed just to avoid compilers from removing m_padding. 577 // This method is needed just to avoid compilers from removing m_padding.
578 uint64_t unusedMethod() const { return m_padding; } 578 uint64_t unusedMethod() const { return m_padding; }
579 579
580 private: 580 private:
581 size_t m_payloadSize; 581 size_t m_payloadSize;
582 uint64_t m_padding; // Preserve 8-byte alignment on 32-bit systems. 582 uint64_t m_padding; // Preserve 8-byte alignment on 32-bit systems.
583 }; 583 };
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
661 class FreePagePool : public PagePool<PageMemory> { 661 class FreePagePool : public PagePool<PageMemory> {
662 public: 662 public:
663 ~FreePagePool(); 663 ~FreePagePool();
664 void addFreePage(int, PageMemory*); 664 void addFreePage(int, PageMemory*);
665 PageMemory* takeFreePage(int); 665 PageMemory* takeFreePage(int);
666 666
667 private: 667 private:
668 Mutex m_mutex[NumberOfHeaps]; 668 Mutex m_mutex[NumberOfHeaps];
669 }; 669 };
670 670
671 class OrphanedPagePool : public PagePool<BasePage> { 671 class OrphanedPagePool : public PagePool<BaseHeapPage> {
672 public: 672 public:
673 ~OrphanedPagePool(); 673 ~OrphanedPagePool();
674 void addOrphanedPage(int, BasePage*); 674 void addOrphanedPage(int, BaseHeapPage*);
675 void decommitOrphanedPages(); 675 void decommitOrphanedPages();
676 #if ENABLE(ASSERT) 676 #if ENABLE(ASSERT)
677 bool contains(void*); 677 bool contains(void*);
678 #endif 678 #endif
679 private: 679 private:
680 void clearMemory(PageMemory*); 680 void clearMemory(PageMemory*);
681 }; 681 };
682 682
683 class FreeList { 683 class FreeList {
684 public: 684 public:
(...skipping 16 matching lines...) Expand all
701 701
702 void getFreeSizeStats(PerBucketFreeListStats bucketStats[], size_t& totalSiz e) const; 702 void getFreeSizeStats(PerBucketFreeListStats bucketStats[], size_t& totalSiz e) const;
703 #endif 703 #endif
704 704
705 private: 705 private:
706 int m_biggestFreeListIndex; 706 int m_biggestFreeListIndex;
707 707
708 // All FreeListEntries in the nth list have size >= 2^n. 708 // All FreeListEntries in the nth list have size >= 2^n.
709 FreeListEntry* m_freeLists[blinkPageSizeLog2]; 709 FreeListEntry* m_freeLists[blinkPageSizeLog2];
710 710
711 friend class NormalPageHeap; 711 friend class ThreadHeapForHeapPage;
712 }; 712 };
713 713
714 // Thread heaps represent a part of the per-thread Blink heap. 714 // Thread heaps represent a part of the per-thread Blink heap.
715 // 715 //
716 // Each Blink thread has a number of thread heaps: one general heap 716 // Each Blink thread has a number of thread heaps: one general heap
717 // that contains any type of object and a number of heaps specialized 717 // that contains any type of object and a number of heaps specialized
718 // for specific object types (such as Node). 718 // for specific object types (such as Node).
719 // 719 //
720 // Each thread heap contains the functionality to allocate new objects 720 // Each thread heap contains the functionality to allocate new objects
721 // (potentially adding new pages to the heap), to find and mark 721 // (potentially adding new pages to the heap), to find and mark
722 // objects during conservative stack scanning and to sweep the set of 722 // objects during conservative stack scanning and to sweep the set of
723 // pages after a GC. 723 // pages after a GC.
724 class PLATFORM_EXPORT BaseHeap { 724 class PLATFORM_EXPORT ThreadHeap {
725 public: 725 public:
726 BaseHeap(ThreadState*, int); 726 ThreadHeap(ThreadState*, int);
727 virtual ~BaseHeap(); 727 virtual ~ThreadHeap();
728 void cleanupPages(); 728 void cleanupPages();
729 729
730 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING) 730 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
731 BasePage* findPageFromAddress(Address); 731 BaseHeapPage* findPageFromAddress(Address);
732 #endif 732 #endif
733 #if ENABLE(GC_PROFILING) 733 #if ENABLE(GC_PROFILING)
734 void snapshot(TracedValue*, ThreadState::SnapshotInfo*); 734 void snapshot(TracedValue*, ThreadState::SnapshotInfo*);
735 void incrementMarkedObjectsAge(); 735 void incrementMarkedObjectsAge();
736 #endif 736 #endif
737 737
738 virtual void clearFreeLists() { } 738 virtual void clearFreeLists() { }
739 void makeConsistentForSweeping(); 739 void makeConsistentForSweeping();
740 #if ENABLE(ASSERT) 740 #if ENABLE(ASSERT)
741 virtual bool isConsistentForSweeping() = 0; 741 virtual bool isConsistentForSweeping() = 0;
742 #endif 742 #endif
743 size_t objectPayloadSizeForTesting(); 743 size_t objectPayloadSizeForTesting();
744 void prepareHeapForTermination(); 744 void prepareHeapForTermination();
745 void prepareForSweep(); 745 void prepareForSweep();
746 Address lazySweep(size_t, size_t gcInfoIndex); 746 Address lazySweep(size_t, size_t gcInfoIndex);
747 void completeSweep(); 747 void completeSweep();
748 748
749 ThreadState* threadState() { return m_threadState; } 749 ThreadState* threadState() { return m_threadState; }
750 int heapIndex() const { return m_index; } 750 int heapIndex() const { return m_index; }
751 inline static size_t allocationSizeFromSize(size_t); 751 inline static size_t allocationSizeFromSize(size_t);
752 inline static size_t roundedAllocationSize(size_t size) 752 inline static size_t roundedAllocationSize(size_t size)
753 { 753 {
754 return allocationSizeFromSize(size) - sizeof(HeapObjectHeader); 754 return allocationSizeFromSize(size) - sizeof(HeapObjectHeader);
755 } 755 }
756 756
757 protected: 757 protected:
758 BasePage* m_firstPage; 758 BaseHeapPage* m_firstPage;
759 BasePage* m_firstUnsweptPage; 759 BaseHeapPage* m_firstUnsweptPage;
760 760
761 private: 761 private:
762 virtual Address lazySweepPages(size_t, size_t gcInfoIndex) = 0; 762 virtual Address lazySweepPages(size_t, size_t gcInfoIndex) = 0;
763 763
764 ThreadState* m_threadState; 764 ThreadState* m_threadState;
765 765
766 // Index into the page pools. This is used to ensure that the pages of the 766 // Index into the page pools. This is used to ensure that the pages of the
767 // same type go into the correct page pool and thus avoid type confusion. 767 // same type go into the correct page pool and thus avoid type confusion.
768 int m_index; 768 int m_index;
769 }; 769 };
770 770
771 class PLATFORM_EXPORT NormalPageHeap final : public BaseHeap { 771 class PLATFORM_EXPORT ThreadHeapForHeapPage final : public ThreadHeap {
772 public: 772 public:
773 NormalPageHeap(ThreadState*, int); 773 ThreadHeapForHeapPage(ThreadState*, int);
774 void addToFreeList(Address address, size_t size) 774 void addToFreeList(Address address, size_t size)
775 { 775 {
776 ASSERT(findPageFromAddress(address)); 776 ASSERT(findPageFromAddress(address));
777 ASSERT(findPageFromAddress(address + size - 1)); 777 ASSERT(findPageFromAddress(address + size - 1));
778 m_freeList.addToFreeList(address, size); 778 m_freeList.addToFreeList(address, size);
779 } 779 }
780 virtual void clearFreeLists() override; 780 virtual void clearFreeLists() override;
781 #if ENABLE(ASSERT) 781 #if ENABLE(ASSERT)
782 virtual bool isConsistentForSweeping() override; 782 virtual bool isConsistentForSweeping() override;
783 bool pagesToBeSweptContains(Address); 783 bool pagesToBeSweptContains(Address);
784 #endif 784 #endif
785 785
786 inline Address allocate(size_t payloadSize, size_t gcInfoIndex); 786 inline Address allocate(size_t payloadSize, size_t gcInfoIndex);
787 inline Address allocateObject(size_t allocationSize, size_t gcInfoIndex); 787 inline Address allocateObject(size_t allocationSize, size_t gcInfoIndex);
788 788
789 void freePage(NormalPage*); 789 void freePage(HeapPage*);
790 790
791 bool coalesce(); 791 bool coalesce();
792 void promptlyFreeObject(HeapObjectHeader*); 792 void promptlyFreeObject(HeapObjectHeader*);
793 bool expandObject(HeapObjectHeader*, size_t); 793 bool expandObject(HeapObjectHeader*, size_t);
794 void shrinkObject(HeapObjectHeader*, size_t); 794 void shrinkObject(HeapObjectHeader*, size_t);
795 void decreasePromptlyFreedSize(size_t size) { m_promptlyFreedSize -= size; } 795 void decreasePromptlyFreedSize(size_t size) { m_promptlyFreedSize -= size; }
796 796
797 #if ENABLE(GC_PROFILING) 797 #if ENABLE(GC_PROFILING)
798 void snapshotFreeList(TracedValue&); 798 void snapshotFreeList(TracedValue&);
799 799
(...skipping 20 matching lines...) Expand all
820 // The size of promptly freed objects in the heap. 820 // The size of promptly freed objects in the heap.
821 size_t m_promptlyFreedSize; 821 size_t m_promptlyFreedSize;
822 822
823 #if ENABLE(GC_PROFILING) 823 #if ENABLE(GC_PROFILING)
824 size_t m_cumulativeAllocationSize; 824 size_t m_cumulativeAllocationSize;
825 size_t m_allocationCount; 825 size_t m_allocationCount;
826 size_t m_inlineAllocationCount; 826 size_t m_inlineAllocationCount;
827 #endif 827 #endif
828 }; 828 };
829 829
830 class LargeObjectHeap final : public BaseHeap { 830 class ThreadHeapForLargeObject final : public ThreadHeap {
831 public: 831 public:
832 LargeObjectHeap(ThreadState*, int); 832 ThreadHeapForLargeObject(ThreadState*, int);
833 Address allocateLargeObjectPage(size_t, size_t gcInfoIndex); 833 Address allocateLargeObject(size_t, size_t gcInfoIndex);
834 void freeLargeObjectPage(LargeObjectPage*); 834 void freeLargeObject(LargeObject*);
835 #if ENABLE(ASSERT) 835 #if ENABLE(ASSERT)
836 virtual bool isConsistentForSweeping() override { return true; } 836 virtual bool isConsistentForSweeping() override { return true; }
837 #endif 837 #endif
838 private: 838 private:
839 Address doAllocateLargeObjectPage(size_t, size_t gcInfoIndex); 839 Address doAllocateLargeObject(size_t, size_t gcInfoIndex);
840 virtual Address lazySweepPages(size_t, size_t gcInfoIndex) override; 840 virtual Address lazySweepPages(size_t, size_t gcInfoIndex) override;
841 }; 841 };
842 842
843 // Mask an address down to the enclosing oilpan heap base page. All oilpan heap 843 // Mask an address down to the enclosing oilpan heap base page. All oilpan heap
844 // pages are aligned at blinkPageBase plus an OS page size. 844 // pages are aligned at blinkPageBase plus an OS page size.
845 // FIXME: Remove PLATFORM_EXPORT once we get a proper public interface to our 845 // FIXME: Remove PLATFORM_EXPORT once we get a proper public interface to our
846 // typed heaps. This is only exported to enable tests in HeapTest.cpp. 846 // typed heaps. This is only exported to enable tests in HeapTest.cpp.
847 PLATFORM_EXPORT inline BasePage* pageFromObject(const void* object) 847 PLATFORM_EXPORT inline BaseHeapPage* pageFromObject(const void* object)
848 { 848 {
849 Address address = reinterpret_cast<Address>(const_cast<void*>(object)); 849 Address address = reinterpret_cast<Address>(const_cast<void*>(object));
850 BasePage* page = reinterpret_cast<BasePage*>(blinkPageAddress(address) + WTF ::kSystemPageSize); 850 BaseHeapPage* page = reinterpret_cast<BaseHeapPage*>(blinkPageAddress(addres s) + WTF::kSystemPageSize);
851 ASSERT(page->contains(address)); 851 ASSERT(page->contains(address));
852 return page; 852 return page;
853 } 853 }
854 854
855 class PLATFORM_EXPORT Heap { 855 class PLATFORM_EXPORT Heap {
856 public: 856 public:
857 static void init(); 857 static void init();
858 static void shutdown(); 858 static void shutdown();
859 static void doShutdown(); 859 static void doShutdown();
860 860
861 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING) 861 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
862 static BasePage* findPageFromAddress(Address); 862 static BaseHeapPage* findPageFromAddress(Address);
863 static BasePage* findPageFromAddress(void* pointer) { return findPageFromAdd ress(reinterpret_cast<Address>(pointer)); } 863 static BaseHeapPage* findPageFromAddress(void* pointer) { return findPageFro mAddress(reinterpret_cast<Address>(pointer)); }
864 static bool containedInHeapOrOrphanedPage(void*); 864 static bool containedInHeapOrOrphanedPage(void*);
865 #endif 865 #endif
866 866
867 // Is the finalizable GC object still alive, but slated for lazy sweeping? 867 // Is the finalizable GC object still alive, but slated for lazy sweeping?
868 // If a lazy sweep is in progress, returns true if the object was found 868 // If a lazy sweep is in progress, returns true if the object was found
869 // to be not reachable during the marking phase, but it has yet to be swept 869 // to be not reachable during the marking phase, but it has yet to be swept
870 // and finalized. The predicate returns false in all other cases. 870 // and finalized. The predicate returns false in all other cases.
871 // 871 //
872 // Holding a reference to an already-dead object is not a valid state 872 // Holding a reference to an already-dead object is not a valid state
873 // to be in; willObjectBeLazilySwept() has undefined behavior if passed 873 // to be in; willObjectBeLazilySwept() has undefined behavior if passed
874 // such a reference. 874 // such a reference.
875 template<typename T> 875 template<typename T>
876 static bool willObjectBeLazilySwept(const T* objectPointer) 876 static bool willObjectBeLazilySwept(const T* objectPointer)
877 { 877 {
878 static_assert(IsGarbageCollectedType<T>::value, "only objects deriving f rom GarbageCollected can be used."); 878 static_assert(IsGarbageCollectedType<T>::value, "only objects deriving f rom GarbageCollected can be used.");
879 #if ENABLE(OILPAN) 879 #if ENABLE(OILPAN)
880 BasePage* page = pageFromObject(objectPointer); 880 BaseHeapPage* page = pageFromObject(objectPointer);
881 if (page->hasBeenSwept()) 881 if (page->hasBeenSwept())
882 return false; 882 return false;
883 ASSERT(page->heap()->threadState()->isSweepingInProgress()); 883 ASSERT(page->heap()->threadState()->isSweepingInProgress());
884 884
885 return !ObjectAliveTrait<T>::isHeapObjectAlive(s_markingVisitor, const_c ast<T*>(objectPointer)); 885 return !ObjectAliveTrait<T>::isHeapObjectAlive(s_markingVisitor, const_c ast<T*>(objectPointer));
886 #else 886 #else
887 // FIXME: remove when lazy sweeping is always on 887 // FIXME: remove when lazy sweeping is always on
888 // (cf. ThreadState::postGCProcessing()). 888 // (cf. ThreadState::postGCProcessing()).
889 return false; 889 return false;
890 #endif 890 #endif
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after
973 // Return true if the last GC found a pointer into a heap page 973 // Return true if the last GC found a pointer into a heap page
974 // during conservative scanning. 974 // during conservative scanning.
975 static bool lastGCWasConservative() { return s_lastGCWasConservative; } 975 static bool lastGCWasConservative() { return s_lastGCWasConservative; }
976 976
977 static FreePagePool* freePagePool() { return s_freePagePool; } 977 static FreePagePool* freePagePool() { return s_freePagePool; }
978 static OrphanedPagePool* orphanedPagePool() { return s_orphanedPagePool; } 978 static OrphanedPagePool* orphanedPagePool() { return s_orphanedPagePool; }
979 979
980 // This look-up uses the region search tree and a negative contains cache to 980 // This look-up uses the region search tree and a negative contains cache to
981 // provide an efficient mapping from arbitrary addresses to the containing 981 // provide an efficient mapping from arbitrary addresses to the containing
982 // heap-page if one exists. 982 // heap-page if one exists.
983 static BasePage* lookup(Address); 983 static BaseHeapPage* lookup(Address);
984 static void addPageMemoryRegion(PageMemoryRegion*); 984 static void addPageMemoryRegion(PageMemoryRegion*);
985 static void removePageMemoryRegion(PageMemoryRegion*); 985 static void removePageMemoryRegion(PageMemoryRegion*);
986 986
987 static const GCInfo* gcInfo(size_t gcInfoIndex) 987 static const GCInfo* gcInfo(size_t gcInfoIndex)
988 { 988 {
989 ASSERT(gcInfoIndex >= 1); 989 ASSERT(gcInfoIndex >= 1);
990 ASSERT(gcInfoIndex < GCInfoTable::maxIndex); 990 ASSERT(gcInfoIndex < GCInfoTable::maxIndex);
991 ASSERT(s_gcInfoTable); 991 ASSERT(s_gcInfoTable);
992 const GCInfo* info = s_gcInfoTable[gcInfoIndex]; 992 const GCInfo* info = s_gcInfoTable[gcInfoIndex];
993 ASSERT(info); 993 ASSERT(info);
(...skipping 268 matching lines...) Expand 10 before | Expand all | Expand 10 after
1262 #define STACK_ALLOCATED() DISALLOW_ALLOCATION() 1262 #define STACK_ALLOCATED() DISALLOW_ALLOCATION()
1263 #define GC_PLUGIN_IGNORE(bug) 1263 #define GC_PLUGIN_IGNORE(bug)
1264 #endif 1264 #endif
1265 1265
1266 NO_SANITIZE_ADDRESS inline 1266 NO_SANITIZE_ADDRESS inline
1267 size_t HeapObjectHeader::size() const 1267 size_t HeapObjectHeader::size() const
1268 { 1268 {
1269 size_t result = m_encoded & headerSizeMask; 1269 size_t result = m_encoded & headerSizeMask;
1270 // Large objects should not refer to header->size(). 1270 // Large objects should not refer to header->size().
1271 // The actual size of a large object is stored in 1271 // The actual size of a large object is stored in
1272 // LargeObjectPage::m_payloadSize. 1272 // LargeObject::m_payloadSize.
1273 ASSERT(result != largeObjectSizeInHeader); 1273 ASSERT(result != largeObjectSizeInHeader);
1274 ASSERT(!pageFromObject(this)->isLargeObjectPage()); 1274 ASSERT(!pageFromObject(this)->isLargeObject());
1275 return result; 1275 return result;
1276 } 1276 }
1277 1277
1278 NO_SANITIZE_ADDRESS 1278 NO_SANITIZE_ADDRESS
1279 void HeapObjectHeader::checkHeader() const 1279 void HeapObjectHeader::checkHeader() const
1280 { 1280 {
1281 ASSERT(pageFromObject(this)->orphaned() || m_magic == magic); 1281 ASSERT(pageFromObject(this)->orphaned() || m_magic == magic);
1282 } 1282 }
1283 1283
1284 Address HeapObjectHeader::payload() 1284 Address HeapObjectHeader::payload()
1285 { 1285 {
1286 return reinterpret_cast<Address>(this) + sizeof(HeapObjectHeader); 1286 return reinterpret_cast<Address>(this) + sizeof(HeapObjectHeader);
1287 } 1287 }
1288 1288
1289 Address HeapObjectHeader::payloadEnd() 1289 Address HeapObjectHeader::payloadEnd()
1290 { 1290 {
1291 return reinterpret_cast<Address>(this) + size(); 1291 return reinterpret_cast<Address>(this) + size();
1292 } 1292 }
1293 1293
1294 NO_SANITIZE_ADDRESS inline 1294 NO_SANITIZE_ADDRESS inline
1295 size_t HeapObjectHeader::payloadSize() 1295 size_t HeapObjectHeader::payloadSize()
1296 { 1296 {
1297 size_t size = m_encoded & headerSizeMask; 1297 size_t size = m_encoded & headerSizeMask;
1298 if (UNLIKELY(size == largeObjectSizeInHeader)) { 1298 if (UNLIKELY(size == largeObjectSizeInHeader)) {
1299 ASSERT(pageFromObject(this)->isLargeObjectPage()); 1299 ASSERT(pageFromObject(this)->isLargeObject());
1300 return static_cast<LargeObjectPage*>(pageFromObject(this))->payloadSize( ); 1300 return static_cast<LargeObject*>(pageFromObject(this))->payloadSize();
1301 } 1301 }
1302 ASSERT(!pageFromObject(this)->isLargeObjectPage()); 1302 ASSERT(!pageFromObject(this)->isLargeObject());
1303 return size - sizeof(HeapObjectHeader); 1303 return size - sizeof(HeapObjectHeader);
1304 } 1304 }
1305 1305
1306 NO_SANITIZE_ADDRESS inline 1306 NO_SANITIZE_ADDRESS inline
1307 bool HeapObjectHeader::isMarked() const 1307 bool HeapObjectHeader::isMarked() const
1308 { 1308 {
1309 checkHeader(); 1309 checkHeader();
1310 return m_encoded & headerMarkBitMask; 1310 return m_encoded & headerMarkBitMask;
1311 } 1311 }
1312 1312
(...skipping 21 matching lines...) Expand all
1334 } 1334 }
1335 1335
1336 NO_SANITIZE_ADDRESS inline 1336 NO_SANITIZE_ADDRESS inline
1337 void HeapObjectHeader::markDead() 1337 void HeapObjectHeader::markDead()
1338 { 1338 {
1339 checkHeader(); 1339 checkHeader();
1340 ASSERT(!isMarked()); 1340 ASSERT(!isMarked());
1341 m_encoded |= headerDeadBitMask; 1341 m_encoded |= headerDeadBitMask;
1342 } 1342 }
1343 1343
1344 size_t BaseHeap::allocationSizeFromSize(size_t size) 1344 size_t ThreadHeap::allocationSizeFromSize(size_t size)
1345 { 1345 {
1346 // Check the size before computing the actual allocation size. The 1346 // Check the size before computing the actual allocation size. The
1347 // allocation size calculation can overflow for large sizes and the check 1347 // allocation size calculation can overflow for large sizes and the check
1348 // therefore has to happen before any calculation on the size. 1348 // therefore has to happen before any calculation on the size.
1349 RELEASE_ASSERT(size < maxHeapObjectSize); 1349 RELEASE_ASSERT(size < maxHeapObjectSize);
1350 1350
1351 // Add space for header. 1351 // Add space for header.
1352 size_t allocationSize = size + sizeof(HeapObjectHeader); 1352 size_t allocationSize = size + sizeof(HeapObjectHeader);
1353 // Align size with allocation granularity. 1353 // Align size with allocation granularity.
1354 allocationSize = (allocationSize + allocationMask) & ~allocationMask; 1354 allocationSize = (allocationSize + allocationMask) & ~allocationMask;
1355 return allocationSize; 1355 return allocationSize;
1356 } 1356 }
1357 1357
1358 Address NormalPageHeap::allocateObject(size_t allocationSize, size_t gcInfoIndex ) 1358 Address ThreadHeapForHeapPage::allocateObject(size_t allocationSize, size_t gcIn foIndex)
1359 { 1359 {
1360 #if ENABLE(GC_PROFILING) 1360 #if ENABLE(GC_PROFILING)
1361 m_cumulativeAllocationSize += allocationSize; 1361 m_cumulativeAllocationSize += allocationSize;
1362 ++m_allocationCount; 1362 ++m_allocationCount;
1363 #endif 1363 #endif
1364 1364
1365 if (LIKELY(allocationSize <= m_remainingAllocationSize)) { 1365 if (LIKELY(allocationSize <= m_remainingAllocationSize)) {
1366 #if ENABLE(GC_PROFILING) 1366 #if ENABLE(GC_PROFILING)
1367 ++m_inlineAllocationCount; 1367 ++m_inlineAllocationCount;
1368 #endif 1368 #endif
1369 Address headerAddress = m_currentAllocationPoint; 1369 Address headerAddress = m_currentAllocationPoint;
1370 m_currentAllocationPoint += allocationSize; 1370 m_currentAllocationPoint += allocationSize;
1371 m_remainingAllocationSize -= allocationSize; 1371 m_remainingAllocationSize -= allocationSize;
1372 ASSERT(gcInfoIndex > 0); 1372 ASSERT(gcInfoIndex > 0);
1373 new (NotNull, headerAddress) HeapObjectHeader(allocationSize, gcInfoInde x); 1373 new (NotNull, headerAddress) HeapObjectHeader(allocationSize, gcInfoInde x);
1374 Address result = headerAddress + sizeof(HeapObjectHeader); 1374 Address result = headerAddress + sizeof(HeapObjectHeader);
1375 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask)); 1375 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
1376 1376
1377 // Unpoison the memory used for the object (payload). 1377 // Unpoison the memory used for the object (payload).
1378 ASAN_UNPOISON_MEMORY_REGION(result, allocationSize - sizeof(HeapObjectHe ader)); 1378 ASAN_UNPOISON_MEMORY_REGION(result, allocationSize - sizeof(HeapObjectHe ader));
1379 FILL_ZERO_IF_NOT_PRODUCTION(result, allocationSize - sizeof(HeapObjectHe ader)); 1379 FILL_ZERO_IF_NOT_PRODUCTION(result, allocationSize - sizeof(HeapObjectHe ader));
1380 ASSERT(findPageFromAddress(headerAddress + allocationSize - 1)); 1380 ASSERT(findPageFromAddress(headerAddress + allocationSize - 1));
1381 return result; 1381 return result;
1382 } 1382 }
1383 return outOfLineAllocate(allocationSize, gcInfoIndex); 1383 return outOfLineAllocate(allocationSize, gcInfoIndex);
1384 } 1384 }
1385 1385
1386 Address NormalPageHeap::allocate(size_t size, size_t gcInfoIndex) 1386 Address ThreadHeapForHeapPage::allocate(size_t size, size_t gcInfoIndex)
1387 { 1387 {
1388 return allocateObject(allocationSizeFromSize(size), gcInfoIndex); 1388 return allocateObject(allocationSizeFromSize(size), gcInfoIndex);
1389 } 1389 }
1390 1390
1391 template<typename T> 1391 template<typename T>
1392 struct HeapIndexTrait { 1392 struct HeapIndexTrait {
1393 static int index() { return NormalPageHeapIndex; }; 1393 static int index() { return GeneralHeap; };
1394 }; 1394 };
1395 1395
1396 // FIXME: The forward declaration is layering violation. 1396 // FIXME: The forward declaration is layering violation.
1397 #define DEFINE_TYPED_HEAP_TRAIT(Type) \ 1397 #define DEFINE_TYPED_HEAP_TRAIT(Type) \
1398 class Type; \ 1398 class Type; \
1399 template<> \ 1399 template<> \
1400 struct HeapIndexTrait<class Type> { \ 1400 struct HeapIndexTrait<class Type> { \
1401 static int index() { return Type##Heap; }; \ 1401 static int index() { return Type##Heap; }; \
1402 }; 1402 };
1403 FOR_EACH_TYPED_HEAP(DEFINE_TYPED_HEAP_TRAIT) 1403 FOR_EACH_TYPED_HEAP(DEFINE_TYPED_HEAP_TRAIT)
1404 #undef DEFINE_TYPED_HEAP_TRAIT 1404 #undef DEFINE_TYPED_HEAP_TRAIT
1405 1405
1406 template<typename T> 1406 template<typename T>
1407 Address Heap::allocateOnHeapIndex(size_t size, int heapIndex, size_t gcInfoIndex ) 1407 Address Heap::allocateOnHeapIndex(size_t size, int heapIndex, size_t gcInfoIndex )
1408 { 1408 {
1409 ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state(); 1409 ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state();
1410 ASSERT(state->isAllocationAllowed()); 1410 ASSERT(state->isAllocationAllowed());
1411 return static_cast<NormalPageHeap*>(state->heap(heapIndex))->allocate(size, gcInfoIndex); 1411 return static_cast<ThreadHeapForHeapPage*>(state->heap(heapIndex))->allocate (size, gcInfoIndex);
1412 } 1412 }
1413 1413
1414 template<typename T> 1414 template<typename T>
1415 Address Heap::allocate(size_t size) 1415 Address Heap::allocate(size_t size)
1416 { 1416 {
1417 return allocateOnHeapIndex<T>(size, HeapIndexTrait<T>::index(), GCInfoTrait< T>::index()); 1417 return allocateOnHeapIndex<T>(size, HeapIndexTrait<T>::index(), GCInfoTrait< T>::index());
1418 } 1418 }
1419 1419
1420 template<typename T> 1420 template<typename T>
1421 Address Heap::reallocate(void* previous, size_t size) 1421 Address Heap::reallocate(void* previous, size_t size)
(...skipping 18 matching lines...) Expand all
1440 memcpy(address, previous, copySize); 1440 memcpy(address, previous, copySize);
1441 return address; 1441 return address;
1442 } 1442 }
1443 1443
1444 class HeapAllocatorQuantizer { 1444 class HeapAllocatorQuantizer {
1445 public: 1445 public:
1446 template<typename T> 1446 template<typename T>
1447 static size_t quantizedSize(size_t count) 1447 static size_t quantizedSize(size_t count)
1448 { 1448 {
1449 RELEASE_ASSERT(count <= kMaxUnquantizedAllocation / sizeof(T)); 1449 RELEASE_ASSERT(count <= kMaxUnquantizedAllocation / sizeof(T));
1450 return BaseHeap::roundedAllocationSize(count * sizeof(T)); 1450 return ThreadHeap::roundedAllocationSize(count * sizeof(T));
1451 } 1451 }
1452 static const size_t kMaxUnquantizedAllocation = maxHeapObjectSize; 1452 static const size_t kMaxUnquantizedAllocation = maxHeapObjectSize;
1453 }; 1453 };
1454 1454
1455 // This is a static-only class used as a trait on collections to make them heap 1455 // This is a static-only class used as a trait on collections to make them heap
1456 // allocated. However see also HeapListHashSetAllocator. 1456 // allocated. However see also HeapListHashSetAllocator.
1457 class HeapAllocator { 1457 class HeapAllocator {
1458 public: 1458 public:
1459 using Quantizer = HeapAllocatorQuantizer; 1459 using Quantizer = HeapAllocatorQuantizer;
1460 using Visitor = blink::Visitor; 1460 using Visitor = blink::Visitor;
1461 static const bool isGarbageCollected = true; 1461 static const bool isGarbageCollected = true;
1462 1462
1463 template <typename T> 1463 template <typename T>
1464 static T* allocateVectorBacking(size_t size) 1464 static T* allocateVectorBacking(size_t size)
1465 { 1465 {
1466 size_t gcInfoIndex = GCInfoTrait<HeapVectorBacking<T, VectorTraits<T>>>: :index(); 1466 size_t gcInfoIndex = GCInfoTrait<HeapVectorBacking<T, VectorTraits<T>>>: :index();
1467 return reinterpret_cast<T*>(Heap::allocateOnHeapIndex<T>(size, VectorHea pIndex, gcInfoIndex)); 1467 return reinterpret_cast<T*>(Heap::allocateOnHeapIndex<T>(size, VectorBac kingHeap, gcInfoIndex));
1468 } 1468 }
1469 PLATFORM_EXPORT static void freeVectorBacking(void* address); 1469 PLATFORM_EXPORT static void freeVectorBacking(void* address);
1470 PLATFORM_EXPORT static bool expandVectorBacking(void*, size_t); 1470 PLATFORM_EXPORT static bool expandVectorBacking(void*, size_t);
1471 static inline bool shrinkVectorBacking(void* address, size_t quantizedCurren tSize, size_t quantizedShrunkSize) 1471 static inline bool shrinkVectorBacking(void* address, size_t quantizedCurren tSize, size_t quantizedShrunkSize)
1472 { 1472 {
1473 shrinkVectorBackingInternal(address, quantizedCurrentSize, quantizedShru nkSize); 1473 shrinkVectorBackingInternal(address, quantizedCurrentSize, quantizedShru nkSize);
1474 return true; 1474 return true;
1475 } 1475 }
1476 template <typename T> 1476 template <typename T>
1477 static T* allocateInlineVectorBacking(size_t size) 1477 static T* allocateInlineVectorBacking(size_t size)
1478 { 1478 {
1479 size_t gcInfoIndex = GCInfoTrait<HeapVectorBacking<T, VectorTraits<T>>>: :index(); 1479 size_t gcInfoIndex = GCInfoTrait<HeapVectorBacking<T, VectorTraits<T>>>: :index();
1480 return reinterpret_cast<T*>(Heap::allocateOnHeapIndex<T>(size, InlineVec torHeapIndex, gcInfoIndex)); 1480 return reinterpret_cast<T*>(Heap::allocateOnHeapIndex<T>(size, InlineVec torBackingHeap, gcInfoIndex));
1481 } 1481 }
1482 PLATFORM_EXPORT static void freeInlineVectorBacking(void* address); 1482 PLATFORM_EXPORT static void freeInlineVectorBacking(void* address);
1483 PLATFORM_EXPORT static bool expandInlineVectorBacking(void*, size_t); 1483 PLATFORM_EXPORT static bool expandInlineVectorBacking(void*, size_t);
1484 static inline bool shrinkInlineVectorBacking(void* address, size_t quantized CurrentSize, size_t quantizedShrinkedSize) 1484 static inline bool shrinkInlineVectorBacking(void* address, size_t quantized CurrentSize, size_t quantizedShrinkedSize)
1485 { 1485 {
1486 shrinkInlineVectorBackingInternal(address, quantizedCurrentSize, quantiz edShrinkedSize); 1486 shrinkInlineVectorBackingInternal(address, quantizedCurrentSize, quantiz edShrinkedSize);
1487 return true; 1487 return true;
1488 } 1488 }
1489 1489
1490 1490
1491 template <typename T, typename HashTable> 1491 template <typename T, typename HashTable>
1492 static T* allocateHashTableBacking(size_t size) 1492 static T* allocateHashTableBacking(size_t size)
1493 { 1493 {
1494 size_t gcInfoIndex = GCInfoTrait<HeapHashTableBacking<HashTable>>::index (); 1494 size_t gcInfoIndex = GCInfoTrait<HeapHashTableBacking<HashTable>>::index ();
1495 return reinterpret_cast<T*>(Heap::allocateOnHeapIndex<T>(size, HashTable HeapIndex, gcInfoIndex)); 1495 return reinterpret_cast<T*>(Heap::allocateOnHeapIndex<T>(size, HashTable BackingHeap, gcInfoIndex));
1496 } 1496 }
1497 template <typename T, typename HashTable> 1497 template <typename T, typename HashTable>
1498 static T* allocateZeroedHashTableBacking(size_t size) 1498 static T* allocateZeroedHashTableBacking(size_t size)
1499 { 1499 {
1500 return allocateHashTableBacking<T, HashTable>(size); 1500 return allocateHashTableBacking<T, HashTable>(size);
1501 } 1501 }
1502 PLATFORM_EXPORT static void freeHashTableBacking(void* address); 1502 PLATFORM_EXPORT static void freeHashTableBacking(void* address);
1503 PLATFORM_EXPORT static bool expandHashTableBacking(void*, size_t); 1503 PLATFORM_EXPORT static bool expandHashTableBacking(void*, size_t);
1504 1504
1505 template <typename Return, typename Metadata> 1505 template <typename Return, typename Metadata>
(...skipping 949 matching lines...) Expand 10 before | Expand all | Expand 10 after
2455 template<typename T, size_t inlineCapacity> 2455 template<typename T, size_t inlineCapacity>
2456 struct GCInfoTrait<HeapVector<T, inlineCapacity>> : public GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator>> { }; 2456 struct GCInfoTrait<HeapVector<T, inlineCapacity>> : public GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator>> { };
2457 template<typename T, size_t inlineCapacity> 2457 template<typename T, size_t inlineCapacity>
2458 struct GCInfoTrait<HeapDeque<T, inlineCapacity>> : public GCInfoTrait<Deque<T, i nlineCapacity, HeapAllocator>> { }; 2458 struct GCInfoTrait<HeapDeque<T, inlineCapacity>> : public GCInfoTrait<Deque<T, i nlineCapacity, HeapAllocator>> { };
2459 template<typename T, typename U, typename V> 2459 template<typename T, typename U, typename V>
2460 struct GCInfoTrait<HeapHashCountedSet<T, U, V>> : public GCInfoTrait<HashCounted Set<T, U, V, HeapAllocator>> { }; 2460 struct GCInfoTrait<HeapHashCountedSet<T, U, V>> : public GCInfoTrait<HashCounted Set<T, U, V, HeapAllocator>> { };
2461 2461
2462 } // namespace blink 2462 } // namespace blink
2463 2463
2464 #endif // Heap_h 2464 #endif // Heap_h
OLDNEW
« no previous file with comments | « no previous file | Source/platform/heap/Heap.cpp » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698