Uses an object pool on x64

runtime/vm/assembler_x64.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #if defined(TARGET_ARCH_X64)
 
 #include "vm/assembler.h"
 #include "vm/heap.h"
 #include "vm/memory_region.h"
@@ skipping 47 matching lines @@
       reinterpret_cast<DetectCPUFeatures>(instructions.EntryPoint())();
   sse4_1_supported_ = (features & kSSE4_1BitMask) != 0;
 #ifdef DEBUG
   initialized_ = true;
 #endif
 }
 
 #undef __
 
 
+Assembler::Assembler(bool use_far_branches)
+    : buffer_(),
+      object_pool_(GrowableObjectArray::Handle()),
+      prologue_offset_(-1),
+      comments_() {
+  // Far branching mode is only needed and implemented for MIPS and ARM.
+  ASSERT(!use_far_branches);
+  if (Isolate::Current() != Dart::vm_isolate()) {
+    object_pool_ = GrowableObjectArray::New(Heap::kOld);
+
+    // These objects and labels need to be accessible through every pool-pointer
+    // at the same index.
+    object_pool_.Add(Object::Handle(Object::null()), Heap::kOld);
+    object_pool_.Add(Bool::True(), Heap::kOld);
+    object_pool_.Add(Bool::False(), Heap::kOld);
+
+    if (StubCode::UpdateStoreBuffer_entry() != NULL) {
+      AddExternalLabel(&StubCode::UpdateStoreBufferLabel(), kNotPatchable);
+    } else {
+      object_pool_.Add(Object::Handle(Object::null()), Heap::kOld);
+    }
+
+    if (StubCode::CallToRuntime_entry() != NULL) {
+      AddExternalLabel(&StubCode::CallToRuntimeLabel(), kNotPatchable);
+    } else {
+      object_pool_.Add(Object::Handle(Object::null()), Heap::kOld);
+    }
+  }
+}
+
+
 void Assembler::InitializeMemoryWithBreakpoints(uword data, int length) {
   memset(reinterpret_cast<void*>(data), Instr::kBreakPointInstruction, length);
 }
 
 
 void Assembler::call(Register reg) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(reg);
   EmitOperandREX(2, operand, REX_NONE);
   EmitUint8(0xFF);
@@ skipping 10 matching lines @@
 
 
 void Assembler::call(Label* label) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   static const int kSize = 5;
   EmitUint8(0xE8);
   EmitLabel(label, kSize);
 }
 
 
+void Assembler::LoadExternalLabel(Register dst,
+                                  const ExternalLabel* label,
+                                  Patchability patchable,
+                                  Register pp) {
+  const int32_t offset =
+      Array::element_offset(AddExternalLabel(label, patchable));
+  LoadWordFromPoolOffset(dst, pp, offset - kHeapObjectTag, patchable);
+}
+
+
 void Assembler::call(const ExternalLabel* label) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
-  intptr_t call_start = buffer_.GetPosition();
 
   // Encode movq(TMP, Immediate(label->address())), but always as imm64.
   EmitRegisterREX(TMP, REX_W);
   EmitUint8(0xB8 | (TMP & 7));
   EmitInt64(label->address());
 
   // Encode call(TMP).
   Operand operand(TMP);
   EmitOperandREX(2, operand, REX_NONE);
   EmitUint8(0xFF);
   EmitOperand(2, operand);
+}
 
+
+void Assembler::CallPatchable(const ExternalLabel* label) {
+  intptr_t call_start = buffer_.GetPosition();
+  LoadExternalLabel(TMP, label, kPatchable, PP);
+  {
+    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+    // Encode call(TMP).
+    Operand operand(TMP);
+    EmitOperandREX(2, operand, REX_NONE);
+    EmitUint8(0xFF);
+    EmitOperand(2, operand);
+  }
   ASSERT((buffer_.GetPosition() - call_start) == kCallExternalLabelSize);
 }
 
 
+void Assembler::CallFromPool(const ExternalLabel* label) {
+  if (Isolate::Current() == Dart::vm_isolate()) {
+    call(label);
+  } else {
+    LoadExternalLabel(TMP, label, kNotPatchable, PP);
+    {

[srdjan, 2013/09/04 22:57:23]

Extra block/scope unnecessary?

[zra, 2013/09/05 00:23:11]

It's needed for EnsureCapacity.

+      AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+      // Encode call(TMP).
+      Operand operand(TMP);
+      EmitOperandREX(2, operand, REX_NONE);
+      EmitUint8(0xFF);
+      EmitOperand(2, operand);
+    }
+  }
+}
+
+
 void Assembler::pushq(Register reg) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitRegisterREX(reg, REX_NONE);
   EmitUint8(0x50 | (reg & 7));
 }
 
 
 void Assembler::pushq(const Address& address) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(6, address, REX_NONE);
@@ skipping 1826 matching lines @@
 
 
 void Assembler::j(Condition condition, const ExternalLabel* label) {
   Label no_jump;
   j(static_cast<Condition>(condition ^ 1), &no_jump);  // Negate condition.
   jmp(label);
   Bind(&no_jump);
 }
 
 
+void Assembler::ConditionalJumpFromPool(Condition condition,
+                                        const ExternalLabel* label,
+                                        Register pp) {
+  Label no_jump;
+  j(static_cast<Condition>(condition ^ 1), &no_jump);  // Negate condition.
+  JumpFromPool(label, pp);
+  Bind(&no_jump);
+}
+
+
 void Assembler::jmp(Register reg) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(reg);
   EmitOperandREX(4, operand, REX_NONE);
   EmitUint8(0xFF);
   EmitOperand(4, operand);
 }
 
 
 void Assembler::jmp(Label* label, bool near) {
@@ skipping 15 matching lines @@
     EmitNearLabelLink(label);
   } else {
     EmitUint8(0xE9);
     EmitLabelLink(label);
   }
 }
 
 
 void Assembler::jmp(const ExternalLabel* label) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
-  intptr_t call_start = buffer_.GetPosition();
 
   // Encode movq(TMP, Immediate(label->address())), but always as imm64.
   EmitRegisterREX(TMP, REX_W);
   EmitUint8(0xB8 | (TMP & 7));
   EmitInt64(label->address());
 
   // Encode jmp(TMP).
   Operand operand(TMP);
   EmitOperandREX(4, operand, REX_NONE);
   EmitUint8(0xFF);
   EmitOperand(4, operand);
+}
 
+
+void Assembler::JumpPatchable(const ExternalLabel* label, Register pp) {
+  intptr_t call_start = buffer_.GetPosition();
+  LoadExternalLabel(TMP, label, kPatchable, pp);
+  {

[srdjan, 2013/09/04 22:57:23]

ditto and elsewhere?

[zra, 2013/09/05 00:23:11]

I've tried to remove, but EnsureCapacity doesn't like it when LoadExternalLabel
is in the same scope with it.

+    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+    // Encode jmp(TMP).
+    Operand operand(TMP);
+    EmitOperandREX(4, operand, REX_NONE);
+    EmitUint8(0xFF);
+    EmitOperand(4, operand);
+  }
   ASSERT((buffer_.GetPosition() - call_start) == kCallExternalLabelSize);
 }
 
 
+void Assembler::JumpFromPool(const ExternalLabel* label, Register pp) {
+  LoadExternalLabel(TMP, label, kNotPatchable, pp);
+  {
+    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+    // Encode jmp(TMP).
+    Operand operand(TMP);
+    EmitOperandREX(4, operand, REX_NONE);
+    EmitUint8(0xFF);
+    EmitOperand(4, operand);
+  }
+}
+
+
 void Assembler::lock() {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitUint8(0xF0);
 }
 
 
 void Assembler::cmpxchgl(const Address& address, Register reg) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(reg, address, REX_NONE);
   EmitUint8(0x0F);
@@ skipping 59 matching lines @@
   if (stack_elements <= 4) {
     for (intptr_t i = 0; i < stack_elements; i++) {
       popq(TMP);
     }
     return;
   }
   addq(RSP, Immediate(stack_elements * kWordSize));
 }
 
 
-void Assembler::LoadObject(Register dst, const Object& object) {
-  if (object.IsSmi() || object.InVMHeap()) {
+int32_t Assembler::AddObject(const Object& obj) {

[srdjan, 2013/09/04 22:57:23]

Why int32_t instead of intptr_t? Length() is intptr_t.

[zra, 2013/09/05 00:23:11]

Done.

+  // The object pool cannot be used in the vm isolate.
+  ASSERT(Isolate::Current() != Dart::vm_isolate());
+  ASSERT(obj.IsNotTemporaryScopedHandle());
+  ASSERT(obj.IsOld());
+  if (object_pool_.IsNull()) {

[srdjan, 2013/09/04 22:57:23]

Should this be an assert? Assembler constructor always allocates an object_pool_
if we are not in VM isolate.

[zra, 2013/09/05 00:23:11]

Done.

+    object_pool_ = GrowableObjectArray::New(Heap::kOld);
+  }
+  for (int i = 0; i < object_pool_.Length(); i++) {
+    if (object_pool_.At(i) == obj.raw()) {

[srdjan, 2013/09/04 22:57:23]

This is slow for large code objects, i.e., compilation time will become
considerably slower for very large functions. This is most likely the reason for
the timeout that you are seeing in the FindCode test.

[zra, 2013/09/05 00:23:11]

Will add a hashmap from obj.raw() to object pool indexes in a future CL.

+      return i;
+    }
+  }
+  object_pool_.Add(obj, Heap::kOld);
+  return object_pool_.Length() - 1;
+}
+
+
+int32_t Assembler::AddExternalLabel(const ExternalLabel* label,
+                                    Patchability patchable) {

[srdjan, 2013/09/04 22:57:23]

s/int32_t/intptr_t/ ?

[zra, 2013/09/05 00:23:11]

Done.

+  // The object pool cannot be used in the vm isolate.
+  ASSERT(Isolate::Current() != Dart::vm_isolate());
+  if (object_pool_.IsNull()) {
+    object_pool_ = GrowableObjectArray::New(Heap::kOld);

[srdjan, 2013/09/04 22:57:23]

Ditto

[zra, 2013/09/05 00:23:11]

Done.

+  }
+  const word address = label->address();

[srdjan, 2013/09/04 22:57:23]

const uword

[zra, 2013/09/05 00:23:11]

Done.

+  ASSERT(Utils::IsAligned(address, 4));
+  // The address is stored in the object array as a RawSmi.
+  const Smi& smi = Smi::Handle(Smi::New(address >> kSmiTagShift));

[srdjan, 2013/09/04 22:57:23]

const Smi& smi = Smi::Handle(reinterpret_cast<RawSmi*>(address)) ?

[zra, 2013/09/05 00:23:11]

Done.

+  if (patchable == kNotPatchable) {
+    // An external label used in a non-patchable call shouldn't also be used in
+    // patchable calls. So, we can re-use existing entries for non-patchable
+    // calls.
+    for (int i = 0; i < object_pool_.Length(); i++) {
+      if (object_pool_.At(i) == smi.raw()) {
+        return i;
+      }
+    }
+  }
+  // If the call is patchable, do not reuse an existing entry since each
+  // reference may be patched independently.
+  object_pool_.Add(smi, Heap::kOld);
+  return object_pool_.Length() - 1;
+}
+
+
+bool Assembler::CanLoadFromObjectPool(const Object& object) {
+  return !object.IsSmi() &&  // Not a Smi
+         // Not in the VMHeap, OR is one of the VMHeap objects we put in every
+         // object pool.
+         (!object.InVMHeap() || (object.raw() == Object::null()) ||
+                                (object.raw() == Bool::True().raw()) ||
+                                (object.raw() == Bool::False().raw())) &&
+         object.IsNotTemporaryScopedHandle() &&
+         object.IsOld();
+}
+
+
+void Assembler::LoadWordFromPoolOffset(Register dst, Register pp,
+                                       int32_t offset, Patchability patchable) {
+  movq(dst, Address(pp, offset));
+  // This sequence must be of fixed size. If offset fits in a signed byte we
+  // have to pad with nops.

[srdjan, 2013/09/04 22:57:23]

Why not generate a movq that always emits long form?

[zra, 2013/09/05 00:23:11]

Done.

+  if (Utils::IsInt(8, offset) && (patchable == kPatchable)) {
+    nop(3);
+  }
+}
+
+
+void Assembler::LoadObjectFromPool(Register dst, const Object& object,
+                           Patchability patchable, Register pp) {
+  if (CanLoadFromObjectPool(object)) {
+    const int32_t offset = Array::element_offset(AddObject(object));
+    LoadWordFromPoolOffset(dst, pp, offset - kHeapObjectTag, patchable);
+  } else {
     movq(dst, Immediate(reinterpret_cast<int64_t>(object.raw())));
-  } else {
-    ASSERT(object.IsNotTemporaryScopedHandle());
-    ASSERT(object.IsOld());
-    AssemblerBuffer::EnsureCapacity ensured(&buffer_);
-    EmitRegisterREX(dst, REX_W);
-    EmitUint8(0xB8 | (dst & 7));
-    buffer_.EmitObject(object);
   }
 }
 
 
 void Assembler::StoreObject(const Address& dst, const Object& object) {
-  if (object.IsSmi() || object.InVMHeap()) {
+  if (CanLoadFromObjectPool(object)) {
+    LoadObjectFromPool(TMP, object, kNotPatchable, PP);
+    movq(dst, TMP);
+  } else {
     movq(dst, Immediate(reinterpret_cast<int64_t>(object.raw())));
-  } else {
-    ASSERT(object.IsNotTemporaryScopedHandle());
-    ASSERT(object.IsOld());
-    LoadObject(TMP, object);
-    movq(dst, TMP);
   }
 }
 
 
 void Assembler::PushObject(const Object& object) {
-  if (object.IsSmi() || object.InVMHeap()) {
+  if (CanLoadFromObjectPool(object)) {
+    LoadObjectFromPool(TMP, object, kNotPatchable, PP);
+    pushq(TMP);
+  } else {
     pushq(Immediate(reinterpret_cast<int64_t>(object.raw())));
-  } else {
-    LoadObject(TMP, object);
-    pushq(TMP);
   }
 }
 
 
 void Assembler::CompareObject(Register reg, const Object& object) {
-  if (object.IsSmi() || object.InVMHeap()) {
+  if (CanLoadFromObjectPool(object)) {
+    ASSERT(reg != TMP);
+    LoadObjectFromPool(TMP, object, kNotPatchable, PP);
+    cmpq(reg, TMP);
+  } else {
     cmpq(reg, Immediate(reinterpret_cast<int64_t>(object.raw())));
-  } else {
-    ASSERT(reg != TMP);
-    LoadObject(TMP, object);
-    cmpq(reg, TMP);
   }
 }
 
 
 // Destroys the value register.
 void Assembler::StoreIntoObjectFilterNoSmi(Register object,
                                            Register value,
                                            Label* no_update) {
   COMPILE_ASSERT((kNewObjectAlignmentOffset == kWordSize) &&
                  (kOldObjectAlignmentOffset == 0), young_alignment);
@@ skipping 42 matching lines @@
   if (can_value_be_smi) {
     StoreIntoObjectFilter(object, value, &done);
   } else {
     StoreIntoObjectFilterNoSmi(object, value, &done);
   }
   // A store buffer update is required.
   if (value != RAX) pushq(RAX);
   if (object != RAX) {
     movq(RAX, object);
   }
-  call(&StubCode::UpdateStoreBufferLabel());
+  CallFromPool(&StubCode::UpdateStoreBufferLabel());
   if (value != RAX) popq(RAX);
   Bind(&done);
 }
 
 
 void Assembler::StoreIntoObjectNoBarrier(Register object,
                                          const Address& dest,
                                          Register value) {
   movq(dest, value);
 #if defined(DEBUG)
@@ skipping 81 matching lines @@
   }
 }
 
 
 void Assembler::LeaveFrame() {
   movq(RSP, RBP);
   popq(RBP);
 }
 
 
+void Assembler::LeaveFrameWithPP() {
+  movq(PP, Address(RBP, -2 * kWordSize));
+  movq(RSP, RBP);
+  popq(RBP);
+}
+
+
 void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
   // Reserve space for arguments and align frame before entering
   // the C++ world.
   AddImmediate(RSP, Immediate(-frame_space));
   if (OS::ActivationFrameAlignment() > 1) {
     andq(RSP, Immediate(~(OS::ActivationFrameAlignment() - 1)));
   }
 }
 
 
@@ skipping 60 matching lines @@
   leave();
 }
 
 
 void Assembler::CallRuntime(const RuntimeEntry& entry,
                             intptr_t argument_count) {
   entry.Call(this, argument_count);
 }
 
 
-void Assembler::EnterDartFrame(intptr_t frame_size) {
+void Assembler::LoadPoolPointer(Register pp) {
+  Label next;
+  call(&next);
+  Bind(&next);
+
+  // Load new pool pointer.
+  const intptr_t object_pool_pc_dist =
+      Instructions::HeaderSize() - Instructions::object_pool_offset() +
+      CodeSize();
+  popq(pp);
+  movq(pp, Address(pp, -object_pool_pc_dist));
+}
+
+
+void Assembler::EnterDartFrame(intptr_t frame_size,
+                               Register new_pp, Register new_pc) {
   EnterFrame(0);
-  Label dart_entry;
-  call(&dart_entry);
-  Bind(&dart_entry);
-  // The runtime system assumes that the code marker address is
-  // kEntryPointToPcMarkerOffset bytes from the entry.  If there is any code
-  // generated before entering the frame, the address needs to be adjusted.
-  const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize();
-  if (offset != 0) {
-    addq(Address(RSP, 0), Immediate(offset));
+  if (new_pc == kNoRegister) {
+    Label dart_entry;
+    call(&dart_entry);
+    Bind(&dart_entry);
+    // The runtime system assumes that the code marker address is
+    // kEntryPointToPcMarkerOffset bytes from the entry.  If there is any code
+    // generated before entering the frame, the address needs to be adjusted.
+    const intptr_t object_pool_pc_dist =
+        Instructions::HeaderSize() - Instructions::object_pool_offset() +
+        CodeSize();
+    const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize();
+    if (offset != 0) {
+      addq(Address(RSP, 0), Immediate(offset));
+    }
+    // Save caller's pool pointer
+    pushq(PP);
+
+    // Load callee's pool pointer.
+    movq(PP, Address(RSP, 1 * kWordSize));
+    movq(PP, Address(PP, -object_pool_pc_dist - offset));
+  } else {
+    pushq(new_pc);
+    pushq(PP);
+    movq(PP, new_pp);
   }
   if (frame_size != 0) {
     subq(RSP, Immediate(frame_size));
   }
 }
 
 
 // On entry to a function compiled for OSR, the caller's frame pointer, the
 // stack locals, and any copied parameters are already in place.  The frame
 // pointer is already set up.  The PC marker is not correct for the
 // optimized function and there may be extra space for spill slots to
 // allocate.
-void Assembler::EnterOsrFrame(intptr_t extra_size) {
-  Label dart_entry;
-  call(&dart_entry);
-  Bind(&dart_entry);
-  // The runtime system assumes that the code marker address is
-  // kEntryPointToPcMarkerOffset bytes from the entry.  Since there is no
-  // code to set up the frame pointer, the address needs to be adjusted.
-  const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize();
-  if (offset != 0) {
-    addq(Address(RSP, 0), Immediate(offset));
+void Assembler::EnterOsrFrame(intptr_t extra_size,
+                              Register new_pp, Register new_pc) {
+  if (new_pc == kNoRegister) {
+    Label dart_entry;
+    call(&dart_entry);
+    Bind(&dart_entry);
+    // The runtime system assumes that the code marker address is
+    // kEntryPointToPcMarkerOffset bytes from the entry.  Since there is no
+    // code to set up the frame pointer, the address needs to be adjusted.
+    const intptr_t object_pool_pc_dist =
+        Instructions::HeaderSize() - Instructions::object_pool_offset() +
+        CodeSize();
+    const intptr_t offset = kEntryPointToPcMarkerOffset - CodeSize();
+    if (offset != 0) {
+      addq(Address(RSP, 0), Immediate(offset));
+    }
+
+    // Load callee's pool pointer.
+    movq(PP, Address(RSP, 0));
+    movq(PP, Address(PP, -object_pool_pc_dist - offset));
+
+    popq(Address(RBP, kPcMarkerSlotFromFp * kWordSize));
+  } else {
+    movq(Address(RBP, kPcMarkerSlotFromFp * kWordSize), new_pc);
+    movq(PP, new_pp);
   }
-  popq(Address(RBP, kPcMarkerSlotFromFp * kWordSize));
   if (extra_size != 0) {
     subq(RSP, Immediate(extra_size));
   }
 }
 
 
 void Assembler::EnterStubFrame() {
   EnterFrame(0);
   pushq(Immediate(0));  // Push 0 in the saved PC area for stub frames.
 }
 
 
+void Assembler::EnterStubFrameWithPP() {
+  EnterFrame(0);
+  pushq(Immediate(0));  // Push 0 in the saved PC area for stub frames.
+  pushq(PP);  // Save caller's pool pointer
+  LoadPoolPointer(PP);
+}
+
+
 void Assembler::TryAllocate(const Class& cls,
                             Label* failure,
                             bool near_jump,
                             Register instance_reg) {
   ASSERT(failure != NULL);
   if (FLAG_inline_alloc) {
     Heap* heap = Isolate::Current()->heap();
     const intptr_t instance_size = cls.instance_size();
     movq(TMP, Immediate(heap->TopAddress()));
     movq(instance_reg, Address(TMP, 0));
@@ skipping 206 matching lines @@
 
 const char* Assembler::FpuRegisterName(FpuRegister reg) {
   ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters));
   return xmm_reg_names[reg];
 }
 
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_X64