Optimise the deoptimisation check to improve performance on modern ARM cores....

src/arm/assembler-arm.cc

 // Copyright (c) 1994-2006 Sun Microsystems Inc.
 // All Rights Reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
 // are met:
 //
 // - Redistributions of source code must retain the above copyright notice,
 // this list of conditions and the following disclaimer.
 //
@@ skipping 303 matching lines @@
     buffer_ = static_cast<byte*>(buffer);
     buffer_size_ = buffer_size;
     own_buffer_ = false;
   }
 
   // Setup buffer pointers.
   ASSERT(buffer_ != NULL);
   pc_ = buffer_;
   reloc_info_writer.Reposition(buffer_ + buffer_size, pc_);
   num_prinfo_ = 0;
+  num_deopt_jump_entries_ = 0;
   next_buffer_check_ = 0;
   const_pool_blocked_nesting_ = 0;
   no_const_pool_before_ = 0;
   last_const_pool_end_ = 0;
   last_bound_pos_ = 0;
   ast_id_for_reloc_info_ = kNoASTId;
 }
 
 
 Assembler::~Assembler() {
@@ skipping 1166 matching lines @@
     svc(kStopCode + code, cond);
   } else {
     svc(kStopCode + kMaxStopCode, cond);
   }
   emit(reinterpret_cast<Instr>(msg));
 #else  // def __arm__
 #ifdef CAN_USE_ARMV5_INSTRUCTIONS
   if (cond != al) {
     Label skip;
     b(&skip, NegateCondition(cond));
-    bkpt(0);
+    bkpt(kBkptStopCode);
     bind(&skip);
   } else {
-    bkpt(0);
+    bkpt(kBkptStopCode);
   }
 #else  // ndef CAN_USE_ARMV5_INSTRUCTIONS
   svc(0x9f0001, cond);
 #endif  // ndef CAN_USE_ARMV5_INSTRUCTIONS
 #endif  // def __arm__
 }
 
 
 void Assembler::bkpt(uint32_t imm16) {  // v5 and above
   ASSERT(is_uint16(imm16));
@@ skipping 868 matching lines @@
 }
 
 
 bool Assembler::IsNop(Instr instr, int type) {
   // Check for mov rx, rx where x = type.
   ASSERT(0 <= type && type <= 14);  // mov pc, pc is not a nop.
   return instr == (al | 13*B21 | type*B12 | type);
 }
 
 
+bool Assembler::IsBkpt(Instr instr, int code) {
+  ASSERT(is_uint16(code));
+  return instr == (al | B24 | B21 | (code >> 4)*B8 | BKPT | (code & 0xf));
+}
+
+
 bool Assembler::ImmediateFitsAddrMode1Instruction(int32_t imm32) {
   uint32_t dummy1;
   uint32_t dummy2;
   return fits_shifter(imm32, &dummy1, &dummy2, NULL);
 }
 
 
 void Assembler::BlockConstPoolFor(int instructions) {
   BlockConstPoolBefore(pc_offset() + instructions * kInstrSize);
 }
@@ skipping 129 matching lines @@
       RelocInfo reloc_info_with_ast_id(pc_, rmode, ast_id_for_reloc_info_);
       ast_id_for_reloc_info_ = kNoASTId;
       reloc_info_writer.Write(&reloc_info_with_ast_id);
     } else {
       reloc_info_writer.Write(&rinfo);
     }
   }
 }
 
 
+void Assembler::RecordDeoptJumpEntry(Address entry, Condition cond) {
+  DeoptJumpEntry deopt_jump_entry(pc_offset(), cond, entry);
+  deopt_jump_entries_[num_deopt_jump_entries_++]= deopt_jump_entry;
+}
+
+
 void Assembler::CheckConstPool(bool force_emit, bool require_jump) {
   // Calculate the offset of the next check. It will be overwritten
   // when a const pool is generated or when const pools are being
   // blocked for a specific range.
   next_buffer_check_ = pc_offset() + kCheckConstInterval;
 
-  // There is nothing to do if there are no pending relocation info entries.
-  if (num_prinfo_ == 0) return;
+  // There is nothing to do if there are no pending relocation info nor
+  // deoptimization entries.
+  if ((num_prinfo_ == 0) && (num_deopt_jump_entries_ == 0)) return;
 
   // We emit a constant pool at regular intervals of about kDistBetweenPools
   // or when requested by parameter force_emit (e.g. after each function).
   // We prefer not to emit a jump unless the max distance is reached or if we
   // are running low on slots, which can happen if a lot of constants are being
   // emitted (e.g. --debug-code and many static references).
-  int dist = pc_offset() - last_const_pool_end_;
+  int jump_instr = require_jump ? kInstrSize : 0;
+  int reloc_info_size = num_prinfo_ * kPointerSize;
+  int deopt_jump_size = num_deopt_jump_entries_ * DeoptJumpEntry::kTotalSize;
+  int needed_space = jump_instr +
+                     kInstrSize +   // For the constant pool marker.
+                     reloc_info_size + deopt_jump_size;
+  int dist = pc_offset() - last_const_pool_end_ + needed_space;
+  // TODO(1236125): Cleanup the "magic" number below. We know that
+  // the code generation will test every kCheckConstIntervalInst.
+  // Thus we are safe as long as we generate less than 7 constant
+  // entries per instruction.
+  int max_dist_at_next_check =
+      dist + kCheckConstIntervalInst * (kInstrSize + 7 * kInstrSize);
+
+  // The distance between the first instruction after the last constant pool
+  // and the end of this constant pool must be less than the addressing range.
   if (!force_emit && dist < kMaxDistBetweenPools &&
       (require_jump || dist < kDistBetweenPools) &&
-      // TODO(1236125): Cleanup the "magic" number below. We know that
-      // the code generation will test every kCheckConstIntervalInst.
-      // Thus we are safe as long as we generate less than 7 constant
-      // entries per instruction.
-      (num_prinfo_ < (kMaxNumPRInfo - (7 * kCheckConstIntervalInst)))) {
+      // We are safe as long as we are certain that we will not generate too
+      // many reloc info or deopt entries in the next kCheckConstIntervalInst.
+      (max_dist_at_next_check < kMaxDistBetweenPools)) {
     return;
   }
 
   // If we did not return by now, we need to emit the constant pool soon.
 
   // However, some small sequences of instructions must not be broken up by the
   // insertion of a constant pool; such sequences are protected by setting
   // either const_pool_blocked_nesting_ or no_const_pool_before_, which are
   // both checked here. Also, recursive calls to CheckConstPool are blocked by
   // no_const_pool_before_.
   if (const_pool_blocked_nesting_ > 0 || pc_offset() < no_const_pool_before_) {
     // Emission is currently blocked; make sure we try again as soon as
     // possible.
     if (const_pool_blocked_nesting_ > 0) {
       next_buffer_check_ = pc_offset() + kInstrSize;
     } else {
       next_buffer_check_ = no_const_pool_before_;
     }
 
     // Something is wrong if emission is forced and blocked at the same time.
     ASSERT(!force_emit);
     return;
   }
 
-  int jump_instr = require_jump ? kInstrSize : 0;
-
   // Check that the code buffer is large enough before emitting the constant
   // pool and relocation information (include the jump over the pool and the
   // constant pool marker).
-  int max_needed_space =
-      jump_instr + kInstrSize + num_prinfo_*(kInstrSize + kMaxRelocSize);
-  while (buffer_space() <= (max_needed_space + kGap)) GrowBuffer();
+  while (buffer_space() <= (needed_space + kGap)) GrowBuffer();
 
   // Block recursive calls to CheckConstPool.
-  BlockConstPoolBefore(pc_offset() + jump_instr + kInstrSize +
-                       num_prinfo_*kInstrSize);
+  BlockConstPoolScope block_const_pool(this);
   // Don't bother to check for the emit calls below.
   next_buffer_check_ = no_const_pool_before_;
 
   // Emit jump over constant pool if necessary.
   Label after_pool;
   if (require_jump) b(&after_pool);
 
   RecordComment("[ Constant Pool");
 
   // Put down constant pool marker "Undefined instruction" as specified by
   // A5.6 (ARMv7) Instruction set encoding.
-  emit(kConstantPoolMarker | num_prinfo_);
+  int constant_pool_size = reloc_info_size + deopt_jump_size;
+  ASSERT(((constant_pool_size / kPointerSize) & ~kConstantPoolLengthMask) == 0);
+  emit(kConstantPoolMarker | (constant_pool_size / kPointerSize));
+
+  int start_of_const_pool = pc_offset();
+  USE(start_of_const_pool);
+
+  // Emit the deoptimization jump table.

[Søren Thygesen Gjesse, 2011/05/16 07:26:39]

This emitting of the deopt-jump table in parts inside the constant pool looks a
bit complicated. Also we didn't used to have code in the constant pool.

How about placing the whole jump table at the end of the code instead of
interleaving it inside the constant pools?

We will have to limit the max size of the generated code for optimized
functions, but I don't think that is a problem.

+  RecordComment("[ Deoptimization jump table");
+  for (int i = 0; i < num_deopt_jump_entries_; i++) {
+    // Patch the code at the deoptimization site.
+    DeoptJumpEntry& deopt_jump_entry = deopt_jump_entries_[i];
+    // Get the offset to the current pc.
+    int new_offset =
+        (pc_offset() - deopt_jump_entry.pc_offset() - kPcLoadDelta);
+    // Compute the location of the deoptimization site.
+    Instr* deopt_site =
+        reinterpret_cast<Instr*>(buffer_ + deopt_jump_entry.pc_offset());
+    // The code to patch is (See LCodeGen::DeoptimizeIf()):
+    // bkpt  kBkptUninitializedCode
+
+    // Check that the instruction to patch is indeed
+    // a bkpt kBkptUninitializedCode.
+    ASSERT(IsBkpt(*deopt_site, kBkptUninitializedCode));
+    // We need to patch the instruction with a branch jumping here.
+    ASSERT((new_offset & 3) == 0);
+    int imm24 = new_offset >> 2;
+    ASSERT(is_int24(imm24));

[Søren Thygesen Gjesse, 2011/05/16 07:26:39]

Please use the CodePatcher class for this.

CodePatcher patcher(pc, 1);
patcher.masm()->b(...)

+    *deopt_site = (deopt_jump_entry.cond() | B27 | B25 | (imm24 & kImm24Mask));
+
+    // Emit the jump to the corresponding deoptimization entry.
+    // We need to manually register this relocation information, because we need
+    // it to be emitted in this constant pool after this jump table.
+    RelocInfo local_rinfo(pc_,
+                          RelocInfo::RUNTIME_ENTRY,
+                          reinterpret_cast<intptr_t>(deopt_jump_entry.entry()));
+    // We are generating instructions in the constant pool.
+    // We are sure the instruction cache will be flushed for these instructions:
+    // the constant pools (and deoptimization jump tables) are intricated in
+    // the generated code, and will be flushed along with it when needed.
+    ldr(pc, MemOperand(pc, 0));
+    prinfo_[num_prinfo_++] = local_rinfo;
+    reloc_info_writer.Write(&local_rinfo);
+  }
+  num_deopt_jump_entries_ = 0;
+  RecordComment("]");
 
   // Emit constant pool entries.
+  RecordComment("[ Constant pool entries");
   for (int i = 0; i < num_prinfo_; i++) {
     RelocInfo& rinfo = prinfo_[i];
     ASSERT(rinfo.rmode() != RelocInfo::COMMENT &&
            rinfo.rmode() != RelocInfo::POSITION &&
            rinfo.rmode() != RelocInfo::STATEMENT_POSITION);
     Instr instr = instr_at(rinfo.pc());
 
     // Instruction to patch must be a ldr/str [pc, #offset].
     // P and U set, B and W clear, Rn == pc, offset12 still 0.
     ASSERT((instr & (7*B25 | P | U | B | W | 15*B16 | kOff12Mask)) ==
            (2*B25 | P | U | pc.code()*B16));
     int delta = pc_ - rinfo.pc() - 8;
     ASSERT(delta >= -4);  // instr could be ldr pc, [pc, #-4] followed by targ32
     if (delta < 0) {
       instr &= ~U;
       delta = -delta;
     }
     ASSERT(is_uint12(delta));
     instr_at_put(rinfo.pc(), instr + delta);
     emit(rinfo.data());
   }
   num_prinfo_ = 0;
+  RecordComment("]");
   last_const_pool_end_ = pc_offset();
 
   RecordComment("]");
 
   if (after_pool.is_linked()) {
     bind(&after_pool);
   }
+  ASSERT(constant_pool_size == (pc_offset() - start_of_const_pool));
 
   // Since a constant pool was just emitted, move the check offset forward by
   // the standard interval.
   next_buffer_check_ = pc_offset() + kCheckConstInterval;
 }
 
 
 } }  // namespace v8::internal
 
 #endif  // V8_TARGET_ARCH_ARM