[arm64] Use exclusive instructions in exchange

test/cctest/test-simulator-arm64.cc

 // Copyright 2017 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 31 matching lines @@
 using namespace v8::base;
 using namespace v8::internal;
 
 #define __ masm.
 
 struct MemoryAccess {
   enum class Kind {
     None,
     Load,
     LoadExcl,
+    LoadAcqExcl,
     Store,
     StoreExcl,
+    StoreRelExcl,
   };
 
   enum class Size {
     Byte,
     HalfWord,
     Word,
   };
 
   MemoryAccess() : kind(Kind::None) {}
   MemoryAccess(Kind kind, Size size, size_t offset, int value = 0)
@@ skipping 38 matching lines @@
 
         case MemoryAccess::Size::Word:
           __ ldr(value_reg, MemOperand(addr_reg));
           break;
       }
       break;
 
     case MemoryAccess::Kind::LoadExcl:
       switch (access.size) {
         case MemoryAccess::Size::Byte:
+          __ ldxrb(value_reg, addr_reg);
+          break;
+
+        case MemoryAccess::Size::HalfWord:
+          __ ldxrh(value_reg, addr_reg);
+          break;
+
+        case MemoryAccess::Size::Word:
+          __ ldxr(value_reg, addr_reg);
+          break;
+      }
+      break;
+
+    case MemoryAccess::Kind::LoadAcqExcl:
+      switch (access.size) {
+        case MemoryAccess::Size::Byte:
           __ ldaxrb(value_reg, addr_reg);
           break;
 
         case MemoryAccess::Size::HalfWord:
           __ ldaxrh(value_reg, addr_reg);
           break;
 
         case MemoryAccess::Size::Word:
           __ ldaxr(value_reg, addr_reg);
           break;
@@ skipping 16 matching lines @@
           __ Mov(value_reg, Operand(access.value));
           __ str(value_reg, MemOperand(addr_reg));
           break;
       }
       break;
 
     case MemoryAccess::Kind::StoreExcl:
       switch (access.size) {
         case MemoryAccess::Size::Byte:
           __ Mov(value_reg, Operand(access.value));
+          __ stxrb(dest_reg, value_reg, addr_reg);
+          break;
+
+        case MemoryAccess::Size::HalfWord:
+          __ Mov(value_reg, Operand(access.value));
+          __ stxrh(dest_reg, value_reg, addr_reg);
+          break;
+
+        case MemoryAccess::Size::Word:
+          __ Mov(value_reg, Operand(access.value));
+          __ stxr(dest_reg, value_reg, addr_reg);
+          break;
+      }
+      break;
+
+    case MemoryAccess::Kind::StoreRelExcl:
+      switch (access.size) {
+        case MemoryAccess::Size::Byte:
+          __ Mov(value_reg, Operand(access.value));
           __ stlxrb(dest_reg, value_reg, addr_reg);
           break;
 
         case MemoryAccess::Size::HalfWord:
           __ Mov(value_reg, Operand(access.value));
           __ stlxrh(dest_reg, value_reg, addr_reg);
           break;
 
         case MemoryAccess::Size::Word:
           __ Mov(value_reg, Operand(access.value));
           __ stlxr(dest_reg, value_reg, addr_reg);
           break;
       }
       break;
   }
 }
 
 static void AssembleLoadExcl(MacroAssembler* assembler, MemoryAccess access,
                              Register value_reg, Register addr_reg) {
-  DCHECK(access.kind == MemoryAccess::Kind::LoadExcl);
+  DCHECK((access.kind == MemoryAccess::Kind::LoadExcl) ||
+         (access.kind == MemoryAccess::Kind::LoadAcqExcl));
   AssembleMemoryAccess(assembler, access, no_reg, value_reg, addr_reg);
 }
 
 static void AssembleStoreExcl(MacroAssembler* assembler, MemoryAccess access,
                               Register dest_reg, Register value_reg,
                               Register addr_reg) {
-  DCHECK(access.kind == MemoryAccess::Kind::StoreExcl);
+  DCHECK((access.kind == MemoryAccess::Kind::StoreExcl) ||
+         (access.kind == MemoryAccess::Kind::StoreRelExcl));
   AssembleMemoryAccess(assembler, access, dest_reg, value_reg, addr_reg);
 }
 
 static void TestInvalidateExclusiveAccess(
     TestData initial_data, MemoryAccess access1, MemoryAccess access2,
     MemoryAccess access3, int expected_res, TestData expected_data) {
   Isolate* isolate = CcTest::i_isolate();
   HandleScope scope(isolate);
   MacroAssembler masm(isolate, NULL, 0, v8::internal::CodeObjectRequired::kYes);
 
   AssembleLoadExcl(&masm, access1, w1, x1);
   AssembleMemoryAccess(&masm, access2, w3, w2, x1);
   AssembleStoreExcl(&masm, access3, w0, w3, x1);
-  __ br(lr);
+  __ Br(lr);
 
   CodeDesc desc;
   masm.GetCode(&desc);
   Handle<Code> code = isolate->factory()->NewCode(
       desc, Code::ComputeFlags(Code::STUB), Handle<Code>());
   TestData t = initial_data;
   Simulator::CallArgument args[] = {
       Simulator::CallArgument(reinterpret_cast<uintptr_t>(&t)),
       Simulator::CallArgument::End()};
   Simulator::current(isolate)->CallVoid(code->entry(), args);
   int res = Simulator::current(isolate)->wreg(0);
 
   CHECK_EQ(expected_res, res);
   switch (access3.size) {
     case MemoryAccess::Size::Byte:
       CHECK_EQ(expected_data.b, t.b);
       break;
 
     case MemoryAccess::Size::HalfWord:
       CHECK_EQ(expected_data.h, t.h);
       break;
 
     case MemoryAccess::Size::Word:
       CHECK_EQ(expected_data.w, t.w);
       break;
   }
 }
 
-TEST(simulator_invalidate_exclusive_access) {
+TEST(simulator_invalidate_acquire_release_exclusive_access) {
   using Kind = MemoryAccess::Kind;
   using Size = MemoryAccess::Size;
 
-  MemoryAccess ldaxr_w(Kind::LoadExcl, Size::Word, offsetof(TestData, w));
-  MemoryAccess stlxr_w(Kind::StoreExcl, Size::Word, offsetof(TestData, w), 7);
+  MemoryAccess ldaxr_w(Kind::LoadAcqExcl, Size::Word, offsetof(TestData, w));
+  MemoryAccess stlxr_w(Kind::StoreRelExcl, Size::Word, offsetof(TestData, w),
+                       7);
 
   // Address mismatch.
   TestInvalidateExclusiveAccess(
       TestData(1), ldaxr_w,
-      MemoryAccess(Kind::LoadExcl, Size::Word, offsetof(TestData, dummy)),
+      MemoryAccess(Kind::LoadAcqExcl, Size::Word, offsetof(TestData, dummy)),
       stlxr_w, 1, TestData(1));
 
   // Size mismatch.
-  TestInvalidateExclusiveAccess(
-      TestData(1), ldaxr_w, MemoryAccess(),
-      MemoryAccess(Kind::StoreExcl, Size::HalfWord, offsetof(TestData, w), 7),
-      1, TestData(1));
+  TestInvalidateExclusiveAccess(TestData(1), ldaxr_w, MemoryAccess(),
+                                MemoryAccess(Kind::StoreRelExcl, Size::HalfWord,
+                                             offsetof(TestData, w), 7),
+                                1, TestData(1));
 
   // Load between ldaxr/stlxr.
   TestInvalidateExclusiveAccess(
       TestData(1), ldaxr_w,
       MemoryAccess(Kind::Load, Size::Word, offsetof(TestData, dummy)), stlxr_w,
       1, TestData(1));
 
   // Store between ldaxr/stlxr.
   TestInvalidateExclusiveAccess(
       TestData(1), ldaxr_w,
       MemoryAccess(Kind::Store, Size::Word, offsetof(TestData, dummy)), stlxr_w,
       1, TestData(1));
 
   // Match
   TestInvalidateExclusiveAccess(TestData(1), ldaxr_w, MemoryAccess(), stlxr_w,
                                 0, TestData(7));
 }
 
+TEST(simulator_invalidate_exclusive_access) {
+  using Kind = MemoryAccess::Kind;
+  using Size = MemoryAccess::Size;
+
+  MemoryAccess ldxr_w(Kind::LoadExcl, Size::Word, offsetof(TestData, w));
+  MemoryAccess stxr_w(Kind::StoreExcl, Size::Word, offsetof(TestData, w), 42);
+
+  // Address mismatch.
+  TestInvalidateExclusiveAccess(
+      TestData(1), ldxr_w,
+      MemoryAccess(Kind::LoadExcl, Size::Word, offsetof(TestData, dummy)),
+      stxr_w, 1, TestData(1));
+
+  // Size mismatch.
+  TestInvalidateExclusiveAccess(
+      TestData(1), ldxr_w, MemoryAccess(),
+      MemoryAccess(Kind::StoreExcl, Size::HalfWord, offsetof(TestData, w), 42),
+      1, TestData(1));
+
+  // Load between ldxr/stxr.
+  TestInvalidateExclusiveAccess(
+      TestData(1), ldxr_w,
+      MemoryAccess(Kind::Load, Size::Word, offsetof(TestData, dummy)), stxr_w,
+      1, TestData(1));
+
+  // Store between ldxr/stxr.
+  TestInvalidateExclusiveAccess(
+      TestData(1), ldxr_w,
+      MemoryAccess(Kind::Store, Size::Word, offsetof(TestData, dummy)), stxr_w,
+      1, TestData(1));
+
+  // Match
+  TestInvalidateExclusiveAccess(TestData(1), ldxr_w, MemoryAccess(), stxr_w, 0,
+                                TestData(42));
+}
+
 static int ExecuteMemoryAccess(Isolate* isolate, TestData* test_data,
                                MemoryAccess access) {
   HandleScope scope(isolate);
   MacroAssembler masm(isolate, NULL, 0, v8::internal::CodeObjectRequired::kYes);
   AssembleMemoryAccess(&masm, access, w0, w2, x1);
   __ br(lr);
 
   CodeDesc desc;
   masm.GetCode(&desc);
   Handle<Code> code = isolate->factory()->NewCode(
@@ skipping 61 matching lines @@
   TestData* test_data_;
   MemoryAccess access_;
   bool is_finished_;
   bool has_request_;
   bool did_request_;
   v8::base::Mutex mutex_;
   v8::base::ConditionVariable has_request_cv_;
   v8::base::ConditionVariable did_request_cv_;
 };
 
-TEST(simulator_invalidate_exclusive_access_threaded) {
+TEST(simulator_invalidate_acquire_release_exclusive_access_threaded) {
   using Kind = MemoryAccess::Kind;
   using Size = MemoryAccess::Size;
 
   Isolate* isolate = CcTest::i_isolate();
   HandleScope scope(isolate);
 
   TestData test_data(1);
 
   MemoryAccessThread thread;
   thread.Start();
 
-  MemoryAccess ldaxr_w(Kind::LoadExcl, Size::Word, offsetof(TestData, w));
-  MemoryAccess stlxr_w(Kind::StoreExcl, Size::Word, offsetof(TestData, w), 7);
+  MemoryAccess ldaxr_w(Kind::LoadAcqExcl, Size::Word, offsetof(TestData, w));
+  MemoryAccess stlxr_w(Kind::StoreRelExcl, Size::Word, offsetof(TestData, w),
+                       7);
 
   // Exclusive store completed by another thread first.
   test_data = TestData(1);
-  thread.NextAndWait(&test_data, MemoryAccess(Kind::LoadExcl, Size::Word,
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::LoadAcqExcl, Size::Word,
                                               offsetof(TestData, w)));
   ExecuteMemoryAccess(isolate, &test_data, ldaxr_w);
-  thread.NextAndWait(&test_data, MemoryAccess(Kind::StoreExcl, Size::Word,
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::StoreRelExcl, Size::Word,
                                               offsetof(TestData, w), 5));
   CHECK_EQ(1, ExecuteMemoryAccess(isolate, &test_data, stlxr_w));
   CHECK_EQ(5, test_data.w);
 
   // Exclusive store completed by another thread; different address, but masked
   // to same
   test_data = TestData(1);
   ExecuteMemoryAccess(isolate, &test_data, ldaxr_w);
-  thread.NextAndWait(&test_data, MemoryAccess(Kind::LoadExcl, Size::Word,
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::LoadAcqExcl, Size::Word,
                                               offsetof(TestData, dummy)));
-  thread.NextAndWait(&test_data, MemoryAccess(Kind::StoreExcl, Size::Word,
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::StoreRelExcl, Size::Word,
                                               offsetof(TestData, dummy), 5));
   CHECK_EQ(1, ExecuteMemoryAccess(isolate, &test_data, stlxr_w));
   CHECK_EQ(1, test_data.w);
 
   // Test failure when store between ldaxr/stlxr.
   test_data = TestData(1);
   ExecuteMemoryAccess(isolate, &test_data, ldaxr_w);
   thread.NextAndWait(&test_data, MemoryAccess(Kind::Store, Size::Word,
                                               offsetof(TestData, dummy)));
   CHECK_EQ(1, ExecuteMemoryAccess(isolate, &test_data, stlxr_w));
   CHECK_EQ(1, test_data.w);
 
   thread.Finish();
   thread.Join();
 }
 
+TEST(simulator_invalidate_exclusive_access_threaded) {
+  using Kind = MemoryAccess::Kind;
+  using Size = MemoryAccess::Size;
+
+  Isolate* isolate = CcTest::i_isolate();
+  HandleScope scope(isolate);
+
+  TestData test_data(1);
+
+  MemoryAccessThread thread;
+  thread.Start();
+
+  MemoryAccess ldxr_w(Kind::LoadExcl, Size::Word, offsetof(TestData, w));
+  MemoryAccess stxr_w(Kind::StoreExcl, Size::Word, offsetof(TestData, w), 7);
+
+  // Exclusive store completed by another thread first.
+  test_data = TestData(1);
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::LoadExcl, Size::Word,
+                                              offsetof(TestData, w)));
+  ExecuteMemoryAccess(isolate, &test_data, ldxr_w);
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::StoreExcl, Size::Word,
+                                              offsetof(TestData, w), 5));
+  CHECK_EQ(1, ExecuteMemoryAccess(isolate, &test_data, stxr_w));
+  CHECK_EQ(5, test_data.w);
+
+  // Exclusive store completed by another thread; different address, but masked
+  // to same
+  test_data = TestData(1);
+  ExecuteMemoryAccess(isolate, &test_data, ldxr_w);
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::LoadExcl, Size::Word,
+                                              offsetof(TestData, dummy)));
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::StoreExcl, Size::Word,
+                                              offsetof(TestData, dummy), 5));
+  CHECK_EQ(1, ExecuteMemoryAccess(isolate, &test_data, stxr_w));
+  CHECK_EQ(1, test_data.w);
+
+  // Test failure when store between ldxr/stxr.
+  test_data = TestData(1);
+  ExecuteMemoryAccess(isolate, &test_data, ldxr_w);
+  thread.NextAndWait(&test_data, MemoryAccess(Kind::Store, Size::Word,
+                                              offsetof(TestData, dummy)));
+  CHECK_EQ(1, ExecuteMemoryAccess(isolate, &test_data, stxr_w));
+  CHECK_EQ(1, test_data.w);
+
+  thread.Finish();
+  thread.Join();
+}
+
 #undef __
 
 #endif  // USE_SIMULATOR