[Atomics] Implement ldaxr/stlxr instructions in ARM64 simulator

src/arm64/simulator-arm64.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <stdlib.h>
 #include <cmath>
 #include <cstdarg>
 
 #if V8_TARGET_ARCH_ARM64
 
@@ skipping 37 matching lines @@
 TEXT_COLOUR clr_flag_value     = FLAG_log_colour ? COLOUR(NORMAL)       : "";
 TEXT_COLOUR clr_reg_name       = FLAG_log_colour ? COLOUR_BOLD(CYAN)    : "";
 TEXT_COLOUR clr_reg_value      = FLAG_log_colour ? COLOUR(CYAN)         : "";
 TEXT_COLOUR clr_fpreg_name     = FLAG_log_colour ? COLOUR_BOLD(MAGENTA) : "";
 TEXT_COLOUR clr_fpreg_value    = FLAG_log_colour ? COLOUR(MAGENTA)      : "";
 TEXT_COLOUR clr_memory_address = FLAG_log_colour ? COLOUR_BOLD(BLUE)    : "";
 TEXT_COLOUR clr_debug_number   = FLAG_log_colour ? COLOUR_BOLD(YELLOW)  : "";
 TEXT_COLOUR clr_debug_message  = FLAG_log_colour ? COLOUR(YELLOW)       : "";
 TEXT_COLOUR clr_printf         = FLAG_log_colour ? COLOUR(GREEN)        : "";
 
+// static
+base::LazyInstance<Simulator::GlobalMonitor>::type Simulator::global_monitor_ =
+    LAZY_INSTANCE_INITIALIZER;
 
 // This is basically the same as PrintF, with a guard for FLAG_trace_sim.
 void Simulator::TraceSim(const char* format, ...) {
   if (FLAG_trace_sim) {
     va_list arguments;
     va_start(arguments, format);
     base::OS::VFPrint(stream_, format, arguments);
     va_end(arguments);
   }
 }
@@ skipping 354 matching lines @@
   // Returning to address 0 exits the Simulator.
   set_lr(kEndOfSimAddress);
 
   // Reset debug helpers.
   breakpoints_.empty();
   break_on_next_ = false;
 }
 
 
 Simulator::~Simulator() {
+  global_monitor_.Pointer()->RemoveProcessor(&global_monitor_processor_);
   delete[] reinterpret_cast<byte*>(stack_);
   if (FLAG_log_instruction_stats) {
     delete instrument_;
   }
   delete disassembler_decoder_;
   delete print_disasm_;
   DeleteArray(last_debugger_input_);
   delete decoder_;
 }
 
@@ skipping 1179 matching lines @@
 
 
 void Simulator::LoadStoreHelper(Instruction* instr,
                                 int64_t offset,
                                 AddrMode addrmode) {
   unsigned srcdst = instr->Rt();
   unsigned addr_reg = instr->Rn();
   uintptr_t address = LoadStoreAddress(addr_reg, offset, addrmode);
   uintptr_t stack = 0;
 
+  base::LockGuard<base::Mutex> lock_guard(&global_monitor_.Pointer()->mutex);
+  if (instr->IsLoad()) {
+    local_monitor_.NotifyLoad(address);
+  } else {
+    local_monitor_.NotifyStore(address);
+    global_monitor_.Pointer()->NotifyStore_Locked(address,
+                                                  &global_monitor_processor_);
+  }
+
   // Handle the writeback for stores before the store. On a CPU the writeback
   // and the store are atomic, but when running on the simulator it is possible
   // to be interrupted in between. The simulator is not thread safe and V8 does
   // not require it to be to run JavaScript therefore the profiler may sample
   // the "simulated" CPU in the middle of load/store with writeback. The code
   // below ensures that push operations are safe even when interrupted: the
   // stack pointer will be decremented before adding an element to the stack.
   if (instr->IsStore()) {
     LoadStoreWriteBack(addr_reg, offset, addrmode);
 
@@ skipping 82 matching lines @@
                                     AddrMode addrmode) {
   unsigned rt = instr->Rt();
   unsigned rt2 = instr->Rt2();
   unsigned addr_reg = instr->Rn();
   size_t access_size = 1 << instr->SizeLSPair();
   int64_t offset = instr->ImmLSPair() * access_size;
   uintptr_t address = LoadStoreAddress(addr_reg, offset, addrmode);
   uintptr_t address2 = address + access_size;
   uintptr_t stack = 0;
 
+  base::LockGuard<base::Mutex> lock_guard(&global_monitor_.Pointer()->mutex);
+  if (instr->IsLoad()) {
+    local_monitor_.NotifyLoad(address);
+    local_monitor_.NotifyLoad(address2);
+  } else {
+    local_monitor_.NotifyStore(address);
+    local_monitor_.NotifyStore(address2);
+    global_monitor_.Pointer()->NotifyStore_Locked(address,
+                                                  &global_monitor_processor_);
+    global_monitor_.Pointer()->NotifyStore_Locked(address2,
+                                                  &global_monitor_processor_);
+  }
+
   // Handle the writeback for stores before the store. On a CPU the writeback
   // and the store are atomic, but when running on the simulator it is possible
   // to be interrupted in between. The simulator is not thread safe and V8 does
   // not require it to be to run JavaScript therefore the profiler may sample
   // the "simulated" CPU in the middle of load/store with writeback. The code
   // below ensures that push operations are safe even when interrupted: the
   // stack pointer will be decremented before adding an element to the stack.
   if (instr->IsStore()) {
     LoadStoreWriteBack(addr_reg, offset, addrmode);
 
@@ skipping 103 matching lines @@
   // Accesses below the stack pointer (but above the platform stack limit) are
   // not allowed in the ABI.
   CheckMemoryAccess(address, stack);
 }
 
 
 void Simulator::VisitLoadLiteral(Instruction* instr) {
   uintptr_t address = instr->LiteralAddress();
   unsigned rt = instr->Rt();
 
+  base::LockGuard<base::Mutex> lock_guard(&global_monitor_.Pointer()->mutex);
+  local_monitor_.NotifyLoad(address);
+
   switch (instr->Mask(LoadLiteralMask)) {
     // Use _no_log variants to suppress the register trace (LOG_REGS,
     // LOG_FP_REGS), then print a more detailed log.
     case LDR_w_lit:
       set_wreg_no_log(rt, MemoryRead<uint32_t>(address));
       LogRead(address, kWRegSize, rt);
       break;
     case LDR_x_lit:
       set_xreg_no_log(rt, MemoryRead<uint64_t>(address));
       LogRead(address, kXRegSize, rt);
@@ skipping 33 matching lines @@
 void Simulator::LoadStoreWriteBack(unsigned addr_reg,
                                    int64_t offset,
                                    AddrMode addrmode) {
   if ((addrmode == PreIndex) || (addrmode == PostIndex)) {
     DCHECK(offset != 0);
     uint64_t address = xreg(addr_reg, Reg31IsStackPointer);
     set_reg(addr_reg, address + offset, Reg31IsStackPointer);
   }
 }
 
+Simulator::TransactionSize Simulator::get_transaction_size(unsigned size) {
+  switch (size) {
+    case 0:
+      return TransactionSize::None;
+    case 1:
+      return TransactionSize::Byte;
+    case 2:
+      return TransactionSize::HalfWord;
+    case 4:
+      return TransactionSize::Word;
+    default:
+      UNREACHABLE();
+  }
+  return TransactionSize::None;
+}
+
 void Simulator::VisitLoadStoreAcquireRelease(Instruction* instr) {
-  // TODO(binji)
+  unsigned rs = instr->Rs();
+  unsigned rt = instr->Rt();
+  unsigned rn = instr->Rn();
+  LoadStoreAcquireReleaseOp op = static_cast<LoadStoreAcquireReleaseOp>(
+      instr->Mask(LoadStoreAcquireReleaseMask));
+  int32_t is_acquire_release = instr->LoadStoreXAcquireRelease();
+  int32_t is_not_exclusive = instr->LoadStoreXNotExclusive();
+  int32_t is_load = instr->LoadStoreXLoad();
+  int32_t is_pair = instr->LoadStoreXPair();
+  DCHECK_NE(is_acquire_release, 0);
+  DCHECK_EQ(is_not_exclusive, 0);  // Non exclusive unimplemented.
+  DCHECK_EQ(is_pair, 0);           // Pair unimplemented.
+  unsigned access_size = 1 << instr->LoadStoreXSizeLog2();
+  uintptr_t address = LoadStoreAddress(rn, 0, AddrMode::Offset);
+  DCHECK(address % access_size == 0);
+  base::LockGuard<base::Mutex> lock_guard(&global_monitor_.Pointer()->mutex);
+  if (is_load != 0) {
+    local_monitor_.NotifyLoadExcl(address, get_transaction_size(access_size));
+    global_monitor_.Pointer()->NotifyLoadExcl_Locked(
+        address, &global_monitor_processor_);
+    switch (op) {
+      case LDAXR_b:
+        set_wreg_no_log(rt, MemoryRead<uint8_t>(address));
+        break;
+      case LDAXR_h:
+        set_wreg_no_log(rt, MemoryRead<uint16_t>(address));
+        break;
+      case LDAXR_w:
+        set_wreg_no_log(rt, MemoryRead<uint32_t>(address));
+        break;
+      default:
+        UNIMPLEMENTED();
+    }
+    LogRead(address, access_size, rt);
+  } else {
+    if (local_monitor_.NotifyStoreExcl(address,
+                                       get_transaction_size(access_size)) &&
+        global_monitor_.Pointer()->NotifyStoreExcl_Locked(
+            address, &global_monitor_processor_)) {
+      switch (op) {
+        case STLXR_b:
+          MemoryWrite<uint8_t>(address, wreg(rt));
+          break;
+        case STLXR_h:
+          MemoryWrite<uint16_t>(address, wreg(rt));
+          break;
+        case STLXR_w:
+          MemoryWrite<uint32_t>(address, wreg(rt));
+          break;
+        default:
+          UNIMPLEMENTED();
+      }
+      LogWrite(address, access_size, rt);
+      set_wreg(rs, 0);
+    } else {
+      set_wreg(rs, 1);
+    }
+  }
 }
 
 void Simulator::CheckMemoryAccess(uintptr_t address, uintptr_t stack) {
   if ((address >= stack_limit_) && (address < stack)) {
     fprintf(stream_, "ACCESS BELOW STACK POINTER:\n");
     fprintf(stream_, "  sp is here:          0x%016" PRIx64 "\n",
             static_cast<uint64_t>(stack));
     fprintf(stream_, "  access was here:     0x%016" PRIx64 "\n",
             static_cast<uint64_t>(address));
     fprintf(stream_, "  stack limit is here: 0x%016" PRIx64 "\n",
@@ skipping 1949 matching lines @@
 
   // The printf parameters are inlined in the code, so skip them.
   set_pc(instr->InstructionAtOffset(kPrintfLength));
 
   // Set LR as if we'd just called a native printf function.
   set_lr(pc());
 
   delete[] format;
 }
 
+Simulator::LocalMonitor::LocalMonitor()
+    : access_state_(MonitorAccess::Open),
+      tagged_addr_(0),
+      size_(TransactionSize::None) {}
+
+void Simulator::LocalMonitor::Clear() {
+  access_state_ = MonitorAccess::Open;
+  tagged_addr_ = 0;
+  size_ = TransactionSize::None;
+}
+
+void Simulator::LocalMonitor::NotifyLoad(uintptr_t addr) {
+  if (access_state_ == MonitorAccess::Exclusive) {
+    // A non exclusive load could clear the local monitor. As a result, it's
+    // most strict to unconditionally clear the local monitor on load.
+    Clear();
+  }
+}
+
+void Simulator::LocalMonitor::NotifyLoadExcl(uintptr_t addr,
+                                             TransactionSize size) {
+  access_state_ = MonitorAccess::Exclusive;
+  tagged_addr_ = addr;
+  size_ = size;
+}
+
+void Simulator::LocalMonitor::NotifyStore(uintptr_t addr) {
+  if (access_state_ == MonitorAccess::Exclusive) {
+    // A non exclusive store could clear the local monitor. As a result, it's
+    // most strict to unconditionally clear the local monitor on store.
+    Clear();
+  }
+}
+
+bool Simulator::LocalMonitor::NotifyStoreExcl(uintptr_t addr,
+                                              TransactionSize size) {
+  if (access_state_ == MonitorAccess::Exclusive) {
+    // It is allowed for a processor to require that the address matches
+    // exactly (B2.10.1), so this comparison does not mask addr.
+    if (addr == tagged_addr_ && size_ == size) {
+      Clear();
+      return true;
+    } else {
+      // It is implementation-defined whether an exclusive store to a
+      // non-tagged address will update memory. As a result, it's most strict
+      // to unconditionally clear the local monitor.
+      Clear();
+      return false;
+    }
+  } else {
+    DCHECK(access_state_ == MonitorAccess::Open);
+    return false;
+  }
+}
+
+Simulator::GlobalMonitor::Processor::Processor()
+    : access_state_(MonitorAccess::Open),
+      tagged_addr_(0),
+      next_(nullptr),
+      prev_(nullptr),
+      failure_counter_(0) {}
+
+void Simulator::GlobalMonitor::Processor::Clear_Locked() {
+  access_state_ = MonitorAccess::Open;
+  tagged_addr_ = 0;
+}
+
+void Simulator::GlobalMonitor::Processor::NotifyLoadExcl_Locked(
+    uintptr_t addr) {
+  access_state_ = MonitorAccess::Exclusive;
+  tagged_addr_ = addr;
+}
+
+void Simulator::GlobalMonitor::Processor::NotifyStore_Locked(
+    uintptr_t addr, bool is_requesting_processor) {
+  if (access_state_ == MonitorAccess::Exclusive) {
+    // A non exclusive store could clear the global monitor. As a result, it's
+    // most strict to unconditionally clear global monitors on store.
+    Clear_Locked();
+  }
+}
+
+bool Simulator::GlobalMonitor::Processor::NotifyStoreExcl_Locked(
+    uintptr_t addr, bool is_requesting_processor) {
+  if (access_state_ == MonitorAccess::Exclusive) {
+    if (is_requesting_processor) {
+      // It is allowed for a processor to require that the address matches
+      // exactly (B2.10.2), so this comparison does not mask addr.
+      if (addr == tagged_addr_) {
+        Clear_Locked();
+        // Introduce occasional stxr failures. This is to simulate the
+        // behavior of hardware, which can randomly fail due to background
+        // cache evictions.
+        if (failure_counter_++ >= kMaxFailureCounter) {
+          failure_counter_ = 0;
+          return false;
+        } else {
+          return true;
+        }
+      }
+    } else if ((addr & kExclusiveTaggedAddrMask) ==
+               (tagged_addr_ & kExclusiveTaggedAddrMask)) {
+      // Check the masked addresses when responding to a successful lock by
+      // another processor so the implementation is more conservative (i.e. the
+      // granularity of locking is as large as possible.)
+      Clear_Locked();
+      return false;
+    }
+  }
+  return false;
+}
+
+Simulator::GlobalMonitor::GlobalMonitor() : head_(nullptr) {}
+
+void Simulator::GlobalMonitor::NotifyLoadExcl_Locked(uintptr_t addr,
+                                                     Processor* processor) {
+  processor->NotifyLoadExcl_Locked(addr);
+  PrependProcessor_Locked(processor);
+}
+
+void Simulator::GlobalMonitor::NotifyStore_Locked(uintptr_t addr,
+                                                  Processor* processor) {
+  // Notify each processor of the store operation.
+  for (Processor* iter = head_; iter; iter = iter->next_) {
+    bool is_requesting_processor = iter == processor;
+    iter->NotifyStore_Locked(addr, is_requesting_processor);
+  }
+}
+
+bool Simulator::GlobalMonitor::NotifyStoreExcl_Locked(uintptr_t addr,
+                                                      Processor* processor) {
+  DCHECK(IsProcessorInLinkedList_Locked(processor));
+  if (processor->NotifyStoreExcl_Locked(addr, true)) {
+    // Notify the other processors that this StoreExcl succeeded.
+    for (Processor* iter = head_; iter; iter = iter->next_) {
+      if (iter != processor) {
+        iter->NotifyStoreExcl_Locked(addr, false);
+      }
+    }
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool Simulator::GlobalMonitor::IsProcessorInLinkedList_Locked(
+    Processor* processor) const {
+  return head_ == processor || processor->next_ || processor->prev_;
+}
+
+void Simulator::GlobalMonitor::PrependProcessor_Locked(Processor* processor) {
+  if (IsProcessorInLinkedList_Locked(processor)) {
+    return;
+  }
+
+  if (head_) {
+    head_->prev_ = processor;
+  }
+  processor->prev_ = nullptr;
+  processor->next_ = head_;
+  head_ = processor;
+}
+
+void Simulator::GlobalMonitor::RemoveProcessor(Processor* processor) {
+  base::LockGuard<base::Mutex> lock_guard(&mutex);
+  if (!IsProcessorInLinkedList_Locked(processor)) {
+    return;
+  }
+
+  if (processor->prev_) {
+    processor->prev_->next_ = processor->next_;
+  } else {
+    head_ = processor->next_;
+  }
+  if (processor->next_) {
+    processor->next_->prev_ = processor->prev_;
+  }
+  processor->prev_ = nullptr;
+  processor->next_ = nullptr;
+}
 
 #endif  // USE_SIMULATOR
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_TARGET_ARCH_ARM64