Reland "ARM64: Add NEON support"

src/arm64/macro-assembler-arm64.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #if V8_TARGET_ARCH_ARM64
 
 #include "src/arm64/frames-arm64.h"
 #include "src/assembler.h"
 #include "src/base/bits.h"
 #include "src/base/division-by-constant.h"
@@ skipping 277 matching lines @@
     dst = rd;
   }
 
   // Copy the result to the system stack pointer.
   if (!dst.Is(rd)) {
     DCHECK(rd.IsSP());
     Assembler::mov(rd, dst);
   }
 }
 
+void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
+  DCHECK(is_uint16(imm));
+  int byte1 = (imm & 0xff);
+  int byte2 = ((imm >> 8) & 0xff);
+  if (byte1 == byte2) {
+    movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
+  } else if (byte1 == 0) {
+    movi(vd, byte2, LSL, 8);
+  } else if (byte2 == 0) {
+    movi(vd, byte1);
+  } else if (byte1 == 0xff) {
+    mvni(vd, ~byte2 & 0xff, LSL, 8);
+  } else if (byte2 == 0xff) {
+    mvni(vd, ~byte1 & 0xff);
+  } else {
+    UseScratchRegisterScope temps(this);
+    Register temp = temps.AcquireW();
+    movz(temp, imm);
+    dup(vd, temp);
+  }
+}
+
+void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
+  DCHECK(is_uint32(imm));
+
+  uint8_t bytes[sizeof(imm)];
+  memcpy(bytes, &imm, sizeof(imm));
+
+  // All bytes are either 0x00 or 0xff.
+  {
+    bool all0orff = true;
+    for (int i = 0; i < 4; ++i) {
+      if ((bytes[i] != 0) && (bytes[i] != 0xff)) {
+        all0orff = false;
+        break;
+      }
+    }
+
+    if (all0orff == true) {
+      movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
+      return;
+    }
+  }
+
+  // Of the 4 bytes, only one byte is non-zero.
+  for (int i = 0; i < 4; i++) {
+    if ((imm & (0xff << (i * 8))) == imm) {
+      movi(vd, bytes[i], LSL, i * 8);
+      return;
+    }
+  }
+
+  // Of the 4 bytes, only one byte is not 0xff.
+  for (int i = 0; i < 4; i++) {
+    uint32_t mask = ~(0xff << (i * 8));
+    if ((imm & mask) == mask) {
+      mvni(vd, ~bytes[i] & 0xff, LSL, i * 8);
+      return;
+    }
+  }
+
+  // Immediate is of the form 0x00MMFFFF.
+  if ((imm & 0xff00ffff) == 0x0000ffff) {
+    movi(vd, bytes[2], MSL, 16);
+    return;
+  }
+
+  // Immediate is of the form 0x0000MMFF.
+  if ((imm & 0xffff00ff) == 0x000000ff) {
+    movi(vd, bytes[1], MSL, 8);
+    return;
+  }
+
+  // Immediate is of the form 0xFFMM0000.
+  if ((imm & 0xff00ffff) == 0xff000000) {
+    mvni(vd, ~bytes[2] & 0xff, MSL, 16);
+    return;
+  }
+  // Immediate is of the form 0xFFFFMM00.
+  if ((imm & 0xffff00ff) == 0xffff0000) {
+    mvni(vd, ~bytes[1] & 0xff, MSL, 8);
+    return;
+  }
+
+  // Top and bottom 16-bits are equal.
+  if (((imm >> 16) & 0xffff) == (imm & 0xffff)) {
+    Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff);
+    return;
+  }
+
+  // Default case.
+  {
+    UseScratchRegisterScope temps(this);
+    Register temp = temps.AcquireW();
+    Mov(temp, imm);
+    dup(vd, temp);
+  }
+}
+
+void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
+  // All bytes are either 0x00 or 0xff.
+  {
+    bool all0orff = true;
+    for (int i = 0; i < 8; ++i) {
+      int byteval = (imm >> (i * 8)) & 0xff;
+      if (byteval != 0 && byteval != 0xff) {
+        all0orff = false;
+        break;
+      }
+    }
+    if (all0orff == true) {
+      movi(vd, imm);
+      return;
+    }
+  }
+
+  // Top and bottom 32-bits are equal.
+  if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) {
+    Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff);
+    return;
+  }
+
+  // Default case.
+  {
+    UseScratchRegisterScope temps(this);
+    Register temp = temps.AcquireX();
+    Mov(temp, imm);
+    if (vd.Is1D()) {
+      mov(vd.D(), 0, temp);
+    } else {
+      dup(vd.V2D(), temp);
+    }
+  }
+}
+
+void MacroAssembler::Movi(const VRegister& vd, uint64_t imm, Shift shift,
+                          int shift_amount) {
+  DCHECK(allow_macro_instructions_);
+  if (shift_amount != 0 || shift != LSL) {
+    movi(vd, imm, shift, shift_amount);
+  } else if (vd.Is8B() || vd.Is16B()) {
+    // 8-bit immediate.
+    DCHECK(is_uint8(imm));
+    movi(vd, imm);
+  } else if (vd.Is4H() || vd.Is8H()) {
+    // 16-bit immediate.
+    Movi16bitHelper(vd, imm);
+  } else if (vd.Is2S() || vd.Is4S()) {
+    // 32-bit immediate.
+    Movi32bitHelper(vd, imm);
+  } else {
+    // 64-bit immediate.
+    Movi64bitHelper(vd, imm);
+  }
+}
+
+void MacroAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
+  // TODO(all): Move 128-bit values in a more efficient way.
+  DCHECK(vd.Is128Bits());
+  UseScratchRegisterScope temps(this);
+  Movi(vd.V2D(), lo);
+  Register temp = temps.AcquireX();
+  Mov(temp, hi);
+  Ins(vd.V2D(), 1, temp);
+}
 
 void MacroAssembler::Mvn(const Register& rd, const Operand& operand) {
   DCHECK(allow_macro_instructions_);
 
   if (operand.NeedsRelocation(this)) {
     Ldr(rd, operand.immediate());
     mvn(rd, rd);
 
   } else if (operand.IsImmediate()) {
     // Call the macro assembler for generic immediates.
@@ skipping 251 matching lines @@
     // The addressing mode is directly supported by the instruction.
     AddSubWithCarry(rd, rn, operand, S, op);
   }
 }
 
 
 void MacroAssembler::LoadStoreMacro(const CPURegister& rt,
                                     const MemOperand& addr,
                                     LoadStoreOp op) {
   int64_t offset = addr.offset();
-  LSDataSize size = CalcLSDataSize(op);
+  unsigned size = CalcLSDataSize(op);
 
   // Check if an immediate offset fits in the immediate field of the
   // appropriate instruction. If not, emit two instructions to perform
   // the operation.
   if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
       !IsImmLSUnscaled(offset)) {
     // Immediate offset that can't be encoded using unsigned or unscaled
     // addressing modes.
     UseScratchRegisterScope temps(this);
     Register temp = temps.AcquireSameSizeAs(addr.base());
@@ skipping 14 matching lines @@
 }
 
 void MacroAssembler::LoadStorePairMacro(const CPURegister& rt,
                                         const CPURegister& rt2,
                                         const MemOperand& addr,
                                         LoadStorePairOp op) {
   // TODO(all): Should we support register offset for load-store-pair?
   DCHECK(!addr.IsRegisterOffset());
 
   int64_t offset = addr.offset();
-  LSDataSize size = CalcLSPairDataSize(op);
+  unsigned size = CalcLSPairDataSize(op);
 
   // Check if the offset fits in the immediate field of the appropriate
   // instruction. If not, emit two instructions to perform the operation.
   if (IsImmLSPair(offset, size)) {
     // Encodable in one load/store pair instruction.
     LoadStorePair(rt, rt2, addr, op);
   } else {
     Register base = addr.base();
     if (addr.IsImmediateOffset()) {
       UseScratchRegisterScope temps(this);
@@ skipping 307 matching lines @@
   DCHECK(dst0.IsValid());
 
   int count = 5 + dst5.IsValid() + dst6.IsValid() + dst7.IsValid();
   int size = dst0.SizeInBytes();
 
   PopHelper(4, size, dst0, dst1, dst2, dst3);
   PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
   PopPostamble(count, size);
 }
 
-
-void MacroAssembler::Push(const Register& src0, const FPRegister& src1) {
+void MacroAssembler::Push(const Register& src0, const VRegister& src1) {
   int size = src0.SizeInBytes() + src1.SizeInBytes();
 
   PushPreamble(size);
   // Reserve room for src0 and push src1.
   str(src1, MemOperand(StackPointer(), -size, PreIndex));
   // Fill the gap with src0.
   str(src0, MemOperand(StackPointer(), src1.SizeInBytes()));
 }
 
 
@@ skipping 446 matching lines @@
     Tst(fpcr, RMode_mask);
     B(eq, &done);
 
     Bind(&unexpected_mode);
     Abort(kUnexpectedFPCRMode);
 
     Bind(&done);
   }
 }
 
-
-void MacroAssembler::CanonicalizeNaN(const FPRegister& dst,
-                                     const FPRegister& src) {
+void MacroAssembler::CanonicalizeNaN(const VRegister& dst,
+                                     const VRegister& src) {
   AssertFPCRState();
 
   // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
   // become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
   // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
   Fsub(dst, src, fp_zero);
 }
 
 
 void MacroAssembler::LoadRoot(CPURegister destination,
@@ skipping 631 matching lines @@
   }
 
   AssertNotSmi(object);
 
   UseScratchRegisterScope temps(this);
   Register temp = temps.AcquireX();
   Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
   JumpIfNotRoot(temp, Heap::kHeapNumberMapRootIndex, on_not_heap_number);
 }
 
-
-void MacroAssembler::TryRepresentDoubleAsInt(Register as_int,
-                                             FPRegister value,
-                                             FPRegister scratch_d,
+void MacroAssembler::TryRepresentDoubleAsInt(Register as_int, VRegister value,
+                                             VRegister scratch_d,
                                              Label* on_successful_conversion,
                                              Label* on_failed_conversion) {
   // Convert to an int and back again, then compare with the original value.
   Fcvtzs(as_int, value);
   Scvtf(scratch_d, as_int);
   Fcmp(value, scratch_d);
 
   if (on_successful_conversion) {
     B(on_successful_conversion, eq);
   }
@@ skipping 593 matching lines @@
     // Drop the execution stack down to the frame pointer and restore
     // the caller frame pointer and return address.
     Mov(jssp, fp);
     AssertStackConsistency();
     Pop(fp, lr);
   }
 }
 
 
 void MacroAssembler::ExitFramePreserveFPRegs() {
-  PushCPURegList(kCallerSavedFP);
+  PushCPURegList(kCallerSavedV);
 }
 
 
 void MacroAssembler::ExitFrameRestoreFPRegs() {
   // Read the registers from the stack without popping them. The stack pointer
   // will be reset as part of the unwinding process.
-  CPURegList saved_fp_regs = kCallerSavedFP;
+  CPURegList saved_fp_regs = kCallerSavedV;
   DCHECK(saved_fp_regs.Count() % 2 == 0);
 
   int offset = ExitFrameConstants::kLastExitFrameField;
   while (!saved_fp_regs.IsEmpty()) {
     const CPURegister& dst0 = saved_fp_regs.PopHighestIndex();
     const CPURegister& dst1 = saved_fp_regs.PopHighestIndex();
     offset -= 2 * kDRegSize;
     Ldp(dst1, dst0, MemOperand(fp, offset));
   }
 }
@@ skipping 458 matching lines @@
            NO_ALLOCATION_FLAGS);
 
   Heap::RootListIndex map_index = mode == MUTABLE
       ? Heap::kMutableHeapNumberMapRootIndex
       : Heap::kHeapNumberMapRootIndex;
 
   // Prepare the heap number map.
   if (!heap_number_map.IsValid()) {
     // If we have a valid value register, use the same type of register to store
     // the map so we can use STP to store both in one instruction.
-    if (value.IsValid() && value.IsFPRegister()) {
+    if (value.IsValid() && value.IsVRegister()) {
       heap_number_map = temps.AcquireD();
     } else {
       heap_number_map = scratch1;
     }
     LoadRoot(heap_number_map, map_index);
   }
   if (emit_debug_code()) {
     Register map;
-    if (heap_number_map.IsFPRegister()) {
+    if (heap_number_map.IsVRegister()) {
       map = scratch1;
       Fmov(map, DoubleRegister(heap_number_map));
     } else {
       map = Register(heap_number_map);
     }
     AssertRegisterIsRoot(map, map_index);
   }
 
   // Store the heap number map and the value in the allocated object.
   if (value.IsSameSizeAndType(heap_number_map)) {
@@ skipping 441 matching lines @@
   const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
   DCHECK(num_unsaved >= 0);
   Claim(num_unsaved);
   PushXRegList(kSafepointSavedRegisters);
 }
 
 
 void MacroAssembler::PushSafepointRegistersAndDoubles() {
   PushSafepointRegisters();
   PushCPURegList(CPURegList(
-      CPURegister::kFPRegister, kDRegSizeInBits,
+      CPURegister::kVRegister, kDRegSizeInBits,
       RegisterConfiguration::Crankshaft()->allocatable_double_codes_mask()));
 }
 
 
 void MacroAssembler::PopSafepointRegistersAndDoubles() {
   PopCPURegList(CPURegList(
-      CPURegister::kFPRegister, kDRegSizeInBits,
+      CPURegister::kVRegister, kDRegSizeInBits,
       RegisterConfiguration::Crankshaft()->allocatable_double_codes_mask()));
   PopSafepointRegisters();
 }
 
 void MacroAssembler::StoreToSafepointRegisterSlot(Register src, Register dst) {
   Poke(src, SafepointRegisterStackIndex(dst.code()) * kPointerSize);
 }
 
 void MacroAssembler::LoadFromSafepointRegisterSlot(Register dst, Register src) {
   Peek(src, SafepointRegisterStackIndex(dst.code()) * kPointerSize);
@@ skipping 531 matching lines @@
   CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
   CPURegister pcs[kPrintfMaxArgCount] = {NoReg, NoReg, NoReg, NoReg};
 
   int arg_count = kPrintfMaxArgCount;
 
   // The PCS varargs registers for printf. Note that x0 is used for the printf
   // format string.
   static const CPURegList kPCSVarargs =
       CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1, arg_count);
   static const CPURegList kPCSVarargsFP =
-      CPURegList(CPURegister::kFPRegister, kDRegSizeInBits, 0, arg_count - 1);
+      CPURegList(CPURegister::kVRegister, kDRegSizeInBits, 0, arg_count - 1);
 
   // We can use caller-saved registers as scratch values, except for the
   // arguments and the PCS registers where they might need to go.
   CPURegList tmp_list = kCallerSaved;
   tmp_list.Remove(x0);      // Used to pass the format string.
   tmp_list.Remove(kPCSVarargs);
   tmp_list.Remove(arg0, arg1, arg2, arg3);
 
-  CPURegList fp_tmp_list = kCallerSavedFP;
+  CPURegList fp_tmp_list = kCallerSavedV;
   fp_tmp_list.Remove(kPCSVarargsFP);
   fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
 
   // Override the MacroAssembler's scratch register list. The lists will be
   // reset automatically at the end of the UseScratchRegisterScope.
   UseScratchRegisterScope temps(this);
   TmpList()->set_list(tmp_list.list());
   FPTmpList()->set_list(fp_tmp_list.list());
 
   // Copies of the printf vararg registers that we can pop from.
   CPURegList pcs_varargs = kPCSVarargs;
   CPURegList pcs_varargs_fp = kPCSVarargsFP;
 
   // Place the arguments. There are lots of clever tricks and optimizations we
   // could use here, but Printf is a debug tool so instead we just try to keep
   // it simple: Move each input that isn't already in the right place to a
   // scratch register, then move everything back.
   for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
     // Work out the proper PCS register for this argument.
     if (args[i].IsRegister()) {
       pcs[i] = pcs_varargs.PopLowestIndex().X();
       // We might only need a W register here. We need to know the size of the
       // argument so we can properly encode it for the simulator call.
       if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
-    } else if (args[i].IsFPRegister()) {
+    } else if (args[i].IsVRegister()) {
       // In C, floats are always cast to doubles for varargs calls.
       pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
     } else {
       DCHECK(args[i].IsNone());
       arg_count = i;
       break;
     }
 
     // If the argument is already in the right place, leave it where it is.
     if (args[i].Aliases(pcs[i])) continue;
 
     // Otherwise, if the argument is in a PCS argument register, allocate an
     // appropriate scratch register and then move it out of the way.
     if (kPCSVarargs.IncludesAliasOf(args[i]) ||
         kPCSVarargsFP.IncludesAliasOf(args[i])) {
       if (args[i].IsRegister()) {
         Register old_arg = Register(args[i]);
         Register new_arg = temps.AcquireSameSizeAs(old_arg);
         Mov(new_arg, old_arg);
         args[i] = new_arg;
       } else {
-        FPRegister old_arg = FPRegister(args[i]);
-        FPRegister new_arg = temps.AcquireSameSizeAs(old_arg);
+        VRegister old_arg = VRegister(args[i]);
+        VRegister new_arg = temps.AcquireSameSizeAs(old_arg);
         Fmov(new_arg, old_arg);
         args[i] = new_arg;
       }
     }
   }
 
   // Do a second pass to move values into their final positions and perform any
   // conversions that may be required.
   for (int i = 0; i < arg_count; i++) {
     DCHECK(pcs[i].type() == args[i].type());
     if (pcs[i].IsRegister()) {
       Mov(Register(pcs[i]), Register(args[i]), kDiscardForSameWReg);
     } else {
-      DCHECK(pcs[i].IsFPRegister());
+      DCHECK(pcs[i].IsVRegister());
       if (pcs[i].SizeInBytes() == args[i].SizeInBytes()) {
-        Fmov(FPRegister(pcs[i]), FPRegister(args[i]));
+        Fmov(VRegister(pcs[i]), VRegister(args[i]));
       } else {
-        Fcvt(FPRegister(pcs[i]), FPRegister(args[i]));
+        Fcvt(VRegister(pcs[i]), VRegister(args[i]));
       }
     }
   }
 
   // Load the format string into x0, as per the procedure-call standard.
   //
   // To make the code as portable as possible, the format string is encoded
   // directly in the instruction stream. It might be cleaner to encode it in a
   // literal pool, but since Printf is usually used for debugging, it is
   // beneficial for it to be minimally dependent on other features.
@@ skipping 67 matching lines @@
   // available as scratch registers until we've preserved them.
   RegList old_tmp_list = TmpList()->list();
   RegList old_fp_tmp_list = FPTmpList()->list();
   TmpList()->set_list(0);
   FPTmpList()->set_list(0);
 
   // Preserve all caller-saved registers as well as NZCV.
   // If csp is the stack pointer, PushCPURegList asserts that the size of each
   // list is a multiple of 16 bytes.
   PushCPURegList(kCallerSaved);
-  PushCPURegList(kCallerSavedFP);
+  PushCPURegList(kCallerSavedV);
 
   // We can use caller-saved registers as scratch values (except for argN).
   CPURegList tmp_list = kCallerSaved;
-  CPURegList fp_tmp_list = kCallerSavedFP;
+  CPURegList fp_tmp_list = kCallerSavedV;
   tmp_list.Remove(arg0, arg1, arg2, arg3);
   fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
   TmpList()->set_list(tmp_list.list());
   FPTmpList()->set_list(fp_tmp_list.list());
 
   { UseScratchRegisterScope temps(this);
     // If any of the arguments are the current stack pointer, allocate a new
     // register for them, and adjust the value to compensate for pushing the
     // caller-saved registers.
     bool arg0_sp = StackPointer().Aliases(arg0);
     bool arg1_sp = StackPointer().Aliases(arg1);
     bool arg2_sp = StackPointer().Aliases(arg2);
     bool arg3_sp = StackPointer().Aliases(arg3);
     if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
       // Allocate a register to hold the original stack pointer value, to pass
       // to PrintfNoPreserve as an argument.
       Register arg_sp = temps.AcquireX();
       Add(arg_sp, StackPointer(),
-          kCallerSaved.TotalSizeInBytes() + kCallerSavedFP.TotalSizeInBytes());
+          kCallerSaved.TotalSizeInBytes() + kCallerSavedV.TotalSizeInBytes());
       if (arg0_sp) arg0 = Register::Create(arg_sp.code(), arg0.SizeInBits());
       if (arg1_sp) arg1 = Register::Create(arg_sp.code(), arg1.SizeInBits());
       if (arg2_sp) arg2 = Register::Create(arg_sp.code(), arg2.SizeInBits());
       if (arg3_sp) arg3 = Register::Create(arg_sp.code(), arg3.SizeInBits());
     }
 
     // Preserve NZCV.
     { UseScratchRegisterScope temps(this);
       Register tmp = temps.AcquireX();
       Mrs(tmp, NZCV);
       Push(tmp, xzr);
     }
 
     PrintfNoPreserve(format, arg0, arg1, arg2, arg3);
 
     // Restore NZCV.
     { UseScratchRegisterScope temps(this);
       Register tmp = temps.AcquireX();
       Pop(xzr, tmp);
       Msr(NZCV, tmp);
     }
   }
 
-  PopCPURegList(kCallerSavedFP);
+  PopCPURegList(kCallerSavedV);
   PopCPURegList(kCallerSaved);
 
   TmpList()->set_list(old_tmp_list);
   FPTmpList()->set_list(old_fp_tmp_list);
 }
 
 
 void MacroAssembler::EmitFrameSetupForCodeAgePatching() {
   // TODO(jbramley): Other architectures use the internal memcpy to copy the
   // sequence. If this is a performance bottleneck, we should consider caching
@@ skipping 93 matching lines @@
   available_->set_list(old_available_);
   availablefp_->set_list(old_availablefp_);
 }
 
 
 Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
   int code = AcquireNextAvailable(available_).code();
   return Register::Create(code, reg.SizeInBits());
 }
 
-
-FPRegister UseScratchRegisterScope::AcquireSameSizeAs(const FPRegister& reg) {
+VRegister UseScratchRegisterScope::AcquireSameSizeAs(const VRegister& reg) {
   int code = AcquireNextAvailable(availablefp_).code();
-  return FPRegister::Create(code, reg.SizeInBits());
+  return VRegister::Create(code, reg.SizeInBits());
 }
 
 
 CPURegister UseScratchRegisterScope::AcquireNextAvailable(
     CPURegList* available) {
   CHECK(!available->IsEmpty());
   CPURegister result = available->PopLowestIndex();
   DCHECK(!AreAliased(result, xzr, csp));
   return result;
 }
@@ skipping 60 matching lines @@
 }
 
 
 #undef __
 
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_TARGET_ARCH_ARM64