Optimise the deoptimisation check to improve performance on modern ARM cores....

src/arm/assembler-arm.cc

Index: src/arm/assembler-arm.cc
===================================================================
--- src/arm/assembler-arm.cc	(revision 7890)
+++ src/arm/assembler-arm.cc	(working copy)
@@ -321,6 +321,7 @@
   pc_ = buffer_;
   reloc_info_writer.Reposition(buffer_ + buffer_size, pc_);
   num_prinfo_ = 0;
+  num_deopt_jump_entries_ = 0;
   next_buffer_check_ = 0;
   const_pool_blocked_nesting_ = 0;
   no_const_pool_before_ = 0;
@@ -1507,10 +1508,10 @@
   if (cond != al) {
     Label skip;
     b(&skip, NegateCondition(cond));
-    bkpt(0);
+    bkpt(kBkptStopCode);
     bind(&skip);
   } else {
-    bkpt(0);
+    bkpt(kBkptStopCode);
   }
 #else  // ndef CAN_USE_ARMV5_INSTRUCTIONS
   svc(0x9f0001, cond);
@@ -2399,6 +2400,12 @@
 }
 
 
+bool Assembler::IsBkpt(Instr instr, int code) {
+  ASSERT(is_uint16(code));
+  return instr == (al | B24 | B21 | (code >> 4)*B8 | BKPT | (code & 0xf));
+}
+
+
 bool Assembler::ImmediateFitsAddrMode1Instruction(int32_t imm32) {
   uint32_t dummy1;
   uint32_t dummy2;
@@ -2548,28 +2555,48 @@
 }
 
 
+void Assembler::RecordDeoptJumpEntry(Address entry, Condition cond) {
+  DeoptJumpEntry deopt_jump_entry(pc_offset(), cond, entry);
+  deopt_jump_entries_[num_deopt_jump_entries_++]= deopt_jump_entry;
+}
+
+
 void Assembler::CheckConstPool(bool force_emit, bool require_jump) {
   // Calculate the offset of the next check. It will be overwritten
   // when a const pool is generated or when const pools are being
   // blocked for a specific range.
   next_buffer_check_ = pc_offset() + kCheckConstInterval;
 
-  // There is nothing to do if there are no pending relocation info entries.
-  if (num_prinfo_ == 0) return;
+  // There is nothing to do if there are no pending relocation info nor
+  // deoptimization entries.
+  if ((num_prinfo_ == 0) && (num_deopt_jump_entries_ == 0)) return;
 
   // We emit a constant pool at regular intervals of about kDistBetweenPools
   // or when requested by parameter force_emit (e.g. after each function).
   // We prefer not to emit a jump unless the max distance is reached or if we
   // are running low on slots, which can happen if a lot of constants are being
   // emitted (e.g. --debug-code and many static references).
-  int dist = pc_offset() - last_const_pool_end_;
+  int jump_instr = require_jump ? kInstrSize : 0;
+  int reloc_info_size = num_prinfo_ * kPointerSize;
+  int deopt_jump_size = num_deopt_jump_entries_ * DeoptJumpEntry::kTotalSize;
+  int needed_space = jump_instr +
+                     kInstrSize +   // For the constant pool marker.
+                     reloc_info_size + deopt_jump_size;
+  int dist = pc_offset() - last_const_pool_end_ + needed_space;
+  // TODO(1236125): Cleanup the "magic" number below. We know that
+  // the code generation will test every kCheckConstIntervalInst.
+  // Thus we are safe as long as we generate less than 7 constant
+  // entries per instruction.
+  int max_dist_at_next_check =
+      dist + kCheckConstIntervalInst * (kInstrSize + 7 * kInstrSize);
+
+  // The distance between the first instruction after the last constant pool
+  // and the end of this constant pool must be less than the addressing range.
   if (!force_emit && dist < kMaxDistBetweenPools &&
       (require_jump || dist < kDistBetweenPools) &&
-      // TODO(1236125): Cleanup the "magic" number below. We know that
-      // the code generation will test every kCheckConstIntervalInst.
-      // Thus we are safe as long as we generate less than 7 constant
-      // entries per instruction.
-      (num_prinfo_ < (kMaxNumPRInfo - (7 * kCheckConstIntervalInst)))) {
+      // We are safe as long as we are certain that we will not generate too
+      // many reloc info or deopt entries in the next kCheckConstIntervalInst.
+      (max_dist_at_next_check < kMaxDistBetweenPools)) {
     return;
   }
 
@@ -2594,18 +2621,13 @@
     return;
   }
 
-  int jump_instr = require_jump ? kInstrSize : 0;
-
   // Check that the code buffer is large enough before emitting the constant
   // pool and relocation information (include the jump over the pool and the
   // constant pool marker).
-  int max_needed_space =
-      jump_instr + kInstrSize + num_prinfo_*(kInstrSize + kMaxRelocSize);
-  while (buffer_space() <= (max_needed_space + kGap)) GrowBuffer();
+  while (buffer_space() <= (needed_space + kGap)) GrowBuffer();
 
   // Block recursive calls to CheckConstPool.
-  BlockConstPoolBefore(pc_offset() + jump_instr + kInstrSize +
-                       num_prinfo_*kInstrSize);
+  BlockConstPoolScope block_const_pool(this);
   // Don't bother to check for the emit calls below.
   next_buffer_check_ = no_const_pool_before_;
 
@@ -2617,9 +2639,55 @@
 
   // Put down constant pool marker "Undefined instruction" as specified by
   // A5.6 (ARMv7) Instruction set encoding.
-  emit(kConstantPoolMarker | num_prinfo_);
+  int constant_pool_size = reloc_info_size + deopt_jump_size;
+  ASSERT(((constant_pool_size / kPointerSize) & ~kConstantPoolLengthMask) == 0);
+  emit(kConstantPoolMarker | (constant_pool_size / kPointerSize));
 
+  int start_of_const_pool = pc_offset();
+  USE(start_of_const_pool);
+
+  // Emit the deoptimization jump table.

[Søren Thygesen Gjesse, 2011/05/16 07:26:39]

This emitting of the deopt-jump table in parts inside the constant pool looks a
bit complicated. Also we didn't used to have code in the constant pool.

How about placing the whole jump table at the end of the code instead of
interleaving it inside the constant pools?

We will have to limit the max size of the generated code for optimized
functions, but I don't think that is a problem.

+  RecordComment("[ Deoptimization jump table");
+  for (int i = 0; i < num_deopt_jump_entries_; i++) {
+    // Patch the code at the deoptimization site.
+    DeoptJumpEntry& deopt_jump_entry = deopt_jump_entries_[i];
+    // Get the offset to the current pc.
+    int new_offset =
+        (pc_offset() - deopt_jump_entry.pc_offset() - kPcLoadDelta);
+    // Compute the location of the deoptimization site.
+    Instr* deopt_site =
+        reinterpret_cast<Instr*>(buffer_ + deopt_jump_entry.pc_offset());
+    // The code to patch is (See LCodeGen::DeoptimizeIf()):
+    // bkpt  kBkptUninitializedCode
+
+    // Check that the instruction to patch is indeed
+    // a bkpt kBkptUninitializedCode.
+    ASSERT(IsBkpt(*deopt_site, kBkptUninitializedCode));
+    // We need to patch the instruction with a branch jumping here.
+    ASSERT((new_offset & 3) == 0);
+    int imm24 = new_offset >> 2;
+    ASSERT(is_int24(imm24));

[Søren Thygesen Gjesse, 2011/05/16 07:26:39]

Please use the CodePatcher class for this.

CodePatcher patcher(pc, 1);
patcher.masm()->b(...)

+    *deopt_site = (deopt_jump_entry.cond() | B27 | B25 | (imm24 & kImm24Mask));
+
+    // Emit the jump to the corresponding deoptimization entry.
+    // We need to manually register this relocation information, because we need
+    // it to be emitted in this constant pool after this jump table.
+    RelocInfo local_rinfo(pc_,
+                          RelocInfo::RUNTIME_ENTRY,
+                          reinterpret_cast<intptr_t>(deopt_jump_entry.entry()));
+    // We are generating instructions in the constant pool.
+    // We are sure the instruction cache will be flushed for these instructions:
+    // the constant pools (and deoptimization jump tables) are intricated in
+    // the generated code, and will be flushed along with it when needed.
+    ldr(pc, MemOperand(pc, 0));
+    prinfo_[num_prinfo_++] = local_rinfo;
+    reloc_info_writer.Write(&local_rinfo);
+  }
+  num_deopt_jump_entries_ = 0;
+  RecordComment("]");
+
   // Emit constant pool entries.
+  RecordComment("[ Constant pool entries");
   for (int i = 0; i < num_prinfo_; i++) {
     RelocInfo& rinfo = prinfo_[i];
     ASSERT(rinfo.rmode() != RelocInfo::COMMENT &&
@@ -2642,6 +2710,7 @@
     emit(rinfo.data());
   }
   num_prinfo_ = 0;
+  RecordComment("]");
   last_const_pool_end_ = pc_offset();
 
   RecordComment("]");
@@ -2649,6 +2718,7 @@
   if (after_pool.is_linked()) {
     bind(&after_pool);
   }
+  ASSERT(constant_pool_size == (pc_offset() - start_of_const_pool));
 
   // Since a constant pool was just emitted, move the check offset forward by
   // the standard interval.