VM: Load allocation-top and -end via Thread.

runtime/vm/intrinsifier_x64.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"  // Needed here to get TARGET_ARCH_X64.
 #if defined(TARGET_ARCH_X64)
 
 #include "vm/intrinsifier.h"
 
 #include "vm/assembler.h"
@@ skipping 54 matching lines @@
   // This snippet of inlined code uses the following registers:
   // RAX, RCX, R13
   // and the newly allocated object is returned in RAX.
   const intptr_t kTypeArgumentsOffset = 2 * kWordSize;
   const intptr_t kArrayOffset = 1 * kWordSize;
   Label fall_through;
 
   // Try allocating in new space.
   const Class& cls = Class::Handle(
       Isolate::Current()->object_store()->growable_object_array_class());
-  __ TryAllocate(cls, &fall_through, Assembler::kFarJump, RAX, kNoRegister);
+  __ TryAllocate(cls, &fall_through, Assembler::kFarJump, RAX,
+                 kNoRegister,  // Pool pointer might not be loaded.
+                 R13);  // temp
 
   // Store backing array object in growable array object.
   __ movq(RCX, Address(RSP, kArrayOffset));  // data argument.
   // RAX is new, no barrier needed.
   __ InitializeFieldNoBarrier(
       RAX,
       FieldAddress(RAX, GrowableObjectArray::data_offset()),
       RCX);
 
   // RAX: new growable array object start as a tagged pointer.
@@ skipping 37 matching lines @@
                      RAX);
   __ LoadObject(RAX, Object::null_object(), PP);
   __ ret();
   __ Bind(&fall_through);
 }
 
 
 #define TYPED_ARRAY_ALLOCATION(type_name, cid, max_len, scale_factor)          \
   Label fall_through;                                                          \
   const intptr_t kArrayLengthStackOffset = 1 * kWordSize;                      \
-  __ MaybeTraceAllocation(cid, &fall_through, false);                          \
+  __ MaybeTraceAllocation(cid, &fall_through, false,                           \
+                          /* inline_isolate = */ false);                       \
   __ movq(RDI, Address(RSP, kArrayLengthStackOffset));  /* Array length. */    \
   /* Check that length is a positive Smi. */                                   \
   /* RDI: requested array length argument. */                                  \
   __ testq(RDI, Immediate(kSmiTagMask));                                       \
   __ j(NOT_ZERO, &fall_through);                                               \
   __ cmpq(RDI, Immediate(0));                                                  \
   __ j(LESS, &fall_through);                                                   \
   __ SmiUntag(RDI);                                                            \
   /* Check for maximum allowed length. */                                      \
   /* RDI: untagged array length. */                                            \
   __ cmpq(RDI, Immediate(max_len));                                            \
   __ j(GREATER, &fall_through);                                                \
   /* Special case for scaling by 16. */                                        \
   if (scale_factor == TIMES_16) {                                              \
     /* double length of array. */                                              \
     __ addq(RDI, RDI);                                                         \
     /* only scale by 8. */                                                     \
     scale_factor = TIMES_8;                                                    \
   }                                                                            \
   const intptr_t fixed_size = sizeof(Raw##type_name) + kObjectAlignment - 1;   \
   __ leaq(RDI, Address(RDI, scale_factor, fixed_size));                        \
   __ andq(RDI, Immediate(-kObjectAlignment));                                  \
-  Heap* heap = Isolate::Current()->heap();                                     \
-  Heap::Space space = heap->SpaceForAllocation(cid);                           \
-  __ movq(RAX, Immediate(heap->TopAddress(space)));                            \
-  __ movq(RAX, Address(RAX, 0));                                               \
+  Heap::Space space = Heap::SpaceForAllocation(cid);                           \
+  __ movq(R13, Address(THR, Thread::heap_offset()));                           \
+  __ movq(RAX, Address(R13, Heap::TopOffset(space)));                          \
   __ movq(RCX, RAX);                                                           \
                                                                                \
   /* RDI: allocation size. */                                                  \
   __ addq(RCX, RDI);                                                           \
   __ j(CARRY, &fall_through);                                                  \
                                                                                \
   /* Check if the allocation fits into the remaining space. */                 \
   /* RAX: potential new object start. */                                       \
   /* RCX: potential next object start. */                                      \
   /* RDI: allocation size. */                                                  \
-  /* R13: scratch register. */                                                 \
-  __ movq(R13, Immediate(heap->EndAddress(space)));                            \
-  __ cmpq(RCX, Address(R13, 0));                                               \
+  /* R13: heap. */                                                             \
+  __ cmpq(RCX, Address(R13, Heap::EndOffset(space)));                          \
   __ j(ABOVE_EQUAL, &fall_through);                                            \
                                                                                \
   /* Successfully allocated the object(s), now update top to point to */       \
   /* next object start and initialize the object. */                           \
-  __ movq(R13, Immediate(heap->TopAddress(space)));                            \
-  __ movq(Address(R13, 0), RCX);                                               \
+  __ movq(Address(R13, Heap::TopOffset(space)), RCX);                          \
   __ addq(RAX, Immediate(kHeapObjectTag));                                     \
-  __ UpdateAllocationStatsWithSize(cid, RDI, space);                           \
+  __ UpdateAllocationStatsWithSize(cid, RDI, space,                            \
+                                   /* inline_isolate = */ false);              \
   /* Initialize the tags. */                                                   \
   /* RAX: new object start as a tagged pointer. */                             \
   /* RCX: new object end address. */                                           \
   /* RDI: allocation size. */                                                  \
   /* R13: scratch register. */                                                 \
   {                                                                            \
     Label size_tag_overflow, done;                                             \
     __ cmpq(RDI, Immediate(RawObject::SizeTag::kMaxSizeTag));                  \
     __ j(ABOVE, &size_tag_overflow, Assembler::kNearJump);                     \
     __ shlq(RDI, Immediate(RawObject::kSizeTagPos - kObjectAlignmentLog2));    \
@@ skipping 1124 matching lines @@
     case Token::kMUL: __ mulsd(XMM0, XMM1); break;
     case Token::kDIV: __ divsd(XMM0, XMM1); break;
     default: UNREACHABLE();
   }
   const Class& double_class = Class::Handle(
       Isolate::Current()->object_store()->double_class());
   __ TryAllocate(double_class,
                  &fall_through,
                  Assembler::kFarJump,
                  RAX,  // Result register.
-                 kNoRegister);  // Pool pointer might not be loaded.
+                 kNoRegister,  // Pool pointer might not be loaded.
+                 R13);  // temp
   __ movsd(FieldAddress(RAX, Double::value_offset()), XMM0);
   __ ret();
   __ Bind(&fall_through);
 }
 
 
 void Intrinsifier::Double_add(Assembler* assembler) {
   DoubleArithmeticOperations(assembler, Token::kADD);
 }
 
@@ skipping 24 matching lines @@
   __ cvtsi2sdq(XMM1, RAX);
   __ movq(RAX, Address(RSP, + 2 * kWordSize));
   __ movsd(XMM0, FieldAddress(RAX, Double::value_offset()));
   __ mulsd(XMM0, XMM1);
   const Class& double_class = Class::Handle(
       Isolate::Current()->object_store()->double_class());
   __ TryAllocate(double_class,
                  &fall_through,
                  Assembler::kFarJump,
                  RAX,  // Result register.
-                 kNoRegister);  // Pool pointer might not be loaded.
+                 kNoRegister,  // Pool pointer might not be loaded.
+                 R13);
   __ movsd(FieldAddress(RAX, Double::value_offset()), XMM0);
   __ ret();
   __ Bind(&fall_through);
 }
 
 
 // Left is double right is integer (Bigint, Mint or Smi)
 void Intrinsifier::DoubleFromInteger(Assembler* assembler) {
   Label fall_through;
   __ movq(RAX, Address(RSP, +1 * kWordSize));
   __ testq(RAX, Immediate(kSmiTagMask));
   __ j(NOT_ZERO, &fall_through);
   // Is Smi.
   __ SmiUntag(RAX);
   __ cvtsi2sdq(XMM0, RAX);
   const Class& double_class = Class::Handle(
       Isolate::Current()->object_store()->double_class());
   __ TryAllocate(double_class,
                  &fall_through,
                  Assembler::kFarJump,
                  RAX,  // Result register.
-                 kNoRegister);  // Pool pointer might not be loaded.
+                 kNoRegister,  // Pool pointer might not be loaded.
+                 R13);
   __ movsd(FieldAddress(RAX, Double::value_offset()), XMM0);
   __ ret();
   __ Bind(&fall_through);
 }
 
 
 void Intrinsifier::Double_getIsNaN(Assembler* assembler) {
   Label is_true;
   __ movq(RAX, Address(RSP, +1 * kWordSize));
   __ movsd(XMM0, FieldAddress(RAX, Double::value_offset()));
@@ skipping 53 matching lines @@
   // Argument is double and is in RAX.
   __ movsd(XMM1, FieldAddress(RAX, Double::value_offset()));
   __ Bind(&double_op);
   __ sqrtsd(XMM0, XMM1);
   const Class& double_class = Class::Handle(
       Isolate::Current()->object_store()->double_class());
   __ TryAllocate(double_class,
                  &fall_through,
                  Assembler::kFarJump,
                  RAX,  // Result register.
-                 kNoRegister);  // Pool pointer might not be loaded.
+                 kNoRegister,  // Pool pointer might not be loaded.
+                 R13);
   __ movsd(FieldAddress(RAX, Double::value_offset()), XMM0);
   __ ret();
   __ Bind(&is_smi);
   __ SmiUntag(RAX);
   __ cvtsi2sdq(XMM1, RAX);
   __ jmp(&double_op);
   __ Bind(&fall_through);
 }
 
 
@@ skipping 253 matching lines @@
 }
 
 
 // Allocates one-byte string of length 'end - start'. The content is not
 // initialized. 'length-reg' contains tagged length.
 // Returns new string as tagged pointer in RAX.
 static void TryAllocateOnebyteString(Assembler* assembler,
                                      Label* ok,
                                      Label* failure,
                                      Register length_reg) {
-  __ MaybeTraceAllocation(kOneByteStringCid, failure, false);
+  __ MaybeTraceAllocation(kOneByteStringCid, failure, false,
+                          /* inline_isolate = */ false);
   if (length_reg != RDI) {
     __ movq(RDI, length_reg);
   }
   Label pop_and_fail;
   __ pushq(RDI);  // Preserve length.
   __ SmiUntag(RDI);
   const intptr_t fixed_size = sizeof(RawString) + kObjectAlignment - 1;
   __ leaq(RDI, Address(RDI, TIMES_1, fixed_size));  // RDI is a Smi.
   __ andq(RDI, Immediate(-kObjectAlignment));
 
-  Isolate* isolate = Isolate::Current();
-  Heap* heap = isolate->heap();
   const intptr_t cid = kOneByteStringCid;
-  Heap::Space space = heap->SpaceForAllocation(cid);
-  __ movq(RAX, Immediate(heap->TopAddress(space)));
-  __ movq(RAX, Address(RAX, 0));
+  Heap::Space space = Heap::SpaceForAllocation(cid);
+  __ movq(R13, Address(THR, Thread::heap_offset()));
+  __ movq(RAX, Address(R13, Heap::TopOffset(space)));
 
   // RDI: allocation size.
   __ movq(RCX, RAX);
   __ addq(RCX, RDI);
   __ j(CARRY,  &pop_and_fail);
 
   // Check if the allocation fits into the remaining space.
   // RAX: potential new object start.
   // RCX: potential next object start.
   // RDI: allocation size.
-  __ movq(R13, Immediate(heap->EndAddress(space)));
-  __ cmpq(RCX, Address(R13, 0));
+  // R13: heap.
+  __ cmpq(RCX, Address(R13, Heap::EndOffset(space)));
   __ j(ABOVE_EQUAL, &pop_and_fail);
 
   // Successfully allocated the object(s), now update top to point to
   // next object start and initialize the object.
-  __ movq(R13, Immediate(heap->TopAddress(space)));
-  __ movq(Address(R13, 0), RCX);
+  __ movq(Address(R13, Heap::TopOffset(space)), RCX);
   __ addq(RAX, Immediate(kHeapObjectTag));
-  __ UpdateAllocationStatsWithSize(cid, RDI, space);
+  __ UpdateAllocationStatsWithSize(cid, RDI, space,
+                                   /* inline_isolate = */ false);
 
   // Initialize the tags.
   // RAX: new object start as a tagged pointer.
   // RDI: allocation size.
   {
     Label size_tag_overflow, done;
     __ cmpq(RDI, Immediate(RawObject::SizeTag::kMaxSizeTag));
     __ j(ABOVE, &size_tag_overflow, Assembler::kNearJump);
     __ shlq(RDI, Immediate(RawObject::kSizeTagPos - kObjectAlignmentLog2));
     __ jmp(&done, Assembler::kNearJump);
@@ skipping 226 matching lines @@
   __ LoadIsolate(RAX);
   __ movq(RAX, Address(RAX, Isolate::current_tag_offset()));
   __ ret();
 }
 
 #undef __
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_X64