[ARM] Implement irregular vector shuffles for SIMD.

src/compiler/arm/code-generator-arm.cc

 // Copyright 2014 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/compiler/code-generator.h"
 
 #include "src/arm/macro-assembler-arm.h"
 #include "src/assembler-inl.h"
 #include "src/compilation-info.h"
 #include "src/compiler/code-generator-impl.h"
@@ skipping 329 matching lines @@
       return pl;
     case kNegative:
       return mi;
     default:
       break;
   }
   UNREACHABLE();
   return kNoCondition;
 }
 
+int GetVtblTableSize(const Simd128Register& src0, const Simd128Register& src1) {
+  // If unary shuffle, table is src0 (2 d-registers).
+  if (src0.is(src1)) return 2;
+  // Binary shuffle, table is src0, src1. They must be consecutive
+  DCHECK_EQ(src0.code() + 1, src1.code());
+  return 4;  // 4 d-registers.
+}
+
 }  // namespace
 
 #define ASSEMBLE_CHECKED_LOAD_FP(Type)                         \
   do {                                                         \
     auto result = i.Output##Type##Register();                  \
     auto offset = i.InputRegister(0);                          \
     if (instr->InputAt(1)->IsRegister()) {                     \
       __ cmp(offset, i.InputRegister(1));                      \
     } else {                                                   \
       __ cmp(offset, i.InputImmediate(1));                     \
@@ skipping 1819 matching lines @@
     }
     case kArmS32x4TransposeLeft: {
       Simd128Register dst = i.OutputSimd128Register(),
                       src1 = i.InputSimd128Register(1);
       DCHECK(dst.is(i.InputSimd128Register(0)));
       // src0 = [0, 1, 2, 3], src1 = [4, 5, 6, 7]
       __ vmov(kScratchQuadReg, src1);
       __ vtrn(Neon32, dst, kScratchQuadReg);  // dst = [0, 4, 2, 6]
       break;
     }
+    case kArmS32x4Shuffle: {
+      Simd128Register dst = i.OutputSimd128Register(),
+                      src0 = i.InputSimd128Register(0),
+                      src1 = i.InputSimd128Register(1);
+      // Check for in-place shuffles.
+      // If dst == src0 == src1, then the shuffle is unary and we only use src0.
+      if (dst.is(src0)) {
+        __ vmov(kScratchQuadReg, src0);
+        src0 = kScratchQuadReg;
+      } else if (dst.is(src1)) {
+        __ vmov(kScratchQuadReg, src1);
+        src1 = kScratchQuadReg;
+      }
+      // Perform shuffle as a vmov per lane.
+      int dst_code = dst.code() * 4;
+      int src0_code = src0.code() * 4;
+      int src1_code = src1.code() * 4;
+      int32_t shuffle = i.InputInt32(2);
+      for (int i = 0; i < 4; i++) {
+        int lane = shuffle & 0x7;
+        int src_code = src0_code;
+        if (lane >= 4) {
+          src_code = src1_code;
+          lane &= 0x3;
+        }
+        __ VmovExtended(dst_code + i, src_code + lane, kScratchReg);

[martyn.capewell, 2017/05/05 14:28:49]

This will become expensive when each s-register move goes via kScratchReg -
core-vfp and vfp-core moves can take longer than core-core or vfp-vfp.

A follow up patch to improve VmovExtended could try using combinations of vdup,
vsli and vsri to eliminate the need for a core scratch register.

e.g. move s40 (a) to s43: d20 = [ X | a ], d21 = [ Y | Z ], use "vsli d21, d20,
#32" to give d21 = [ a | Z ].

[bbudge, 2017/05/05 20:36:28]

Yes, I think I have a TODO to improve VMovExtended. It's also potentially used a
lot for gap moves. I'll get on that.

+        shuffle >>= 8;
+      }
+      break;
+    }
     case kArmS32x4TransposeRight: {
       Simd128Register dst = i.OutputSimd128Register(),
                       src1 = i.InputSimd128Register(1);
       DCHECK(dst.is(i.InputSimd128Register(0)));
       // src0 = [4, 5, 6, 7], src1 = [0, 1, 2, 3] (flipped from TransposeLeft).
       __ vmov(kScratchQuadReg, src1);
       __ vtrn(Neon32, kScratchQuadReg, dst);  // dst = [1, 5, 3, 7]
       break;
     }
     case kArmS16x8ZipLeft: {
@@ skipping 43 matching lines @@
     }
     case kArmS16x8TransposeRight: {
       Simd128Register dst = i.OutputSimd128Register(),
                       src1 = i.InputSimd128Register(1);
       DCHECK(dst.is(i.InputSimd128Register(0)));
       // src0 = [8, 9, 10, 11, ... 15], src1 = [0, 1, 2, 3, ... 7] (flipped).
       __ vmov(kScratchQuadReg, src1);
       __ vtrn(Neon16, kScratchQuadReg, dst);  // dst = [1, 9, 3, 11, ... 15]
       break;
     }
+    case kArmS16x8Shuffle: {
+      Simd128Register dst = i.OutputSimd128Register(),
+                      src0 = i.InputSimd128Register(0),
+                      src1 = i.InputSimd128Register(1);
+      DwVfpRegister table_base = src0.low();
+      int table_size = GetVtblTableSize(src0, src1);
+      // Convert the shuffle lane masks to byte masks in kScratchQuadReg.
+      int scratch_s_base = kScratchQuadReg.code() * 4;
+      for (int j = 0; j < 2; j++) {
+        int32_t four_lanes = i.InputInt32(2 + j);
+        for (int k = 0; k < 2; k++) {
+          uint8_t w0 = (four_lanes & 0xFF) * kShortSize;
+          four_lanes >>= 8;
+          uint8_t w1 = (four_lanes & 0xFF) * kShortSize;
+          four_lanes >>= 8;
+          int32_t mask = w0 | ((w0 + 1) << 8) | (w1 << 16) | ((w1 + 1) << 24);
+          __ vmov(SwVfpRegister::from_code(scratch_s_base + 2 * j + k),
+                  bit_cast<float>(mask));

[martyn.capewell, 2017/05/05 14:28:49]

This may cause a problem - if your mask looks like a signalling NaN, it could be
quietened on passing it to vmov. I had a similar problem in the ARM64 simulator.
The outcome was some of the test results differed between hardware and
simulation.

A vmov-immediate function that accepts uint32_t would help.

[bbudge, 2017/05/05 20:36:28]

I could also finesse the code so NaNs can't be generated. We can guarantee that
byte indices are in [0, 31] - but I'll have to mask them above. This way there's
a very slight chance (depending on the mask) that we can leverage the more
efficient encoding this move supports.

[martyn.capewell, 2017/05/08 13:33:53]

That will fix the NaN problem. However, the assembler doesn't support some of
the other non-FP vmov immediate forms that might be useful in this context. See
"A7.4.6 One register and a modified immediate value" in the ARM ARM for more
info.

Not for this patch, though.

+        }
+      }
+      NeonListOperand table(table_base, table_size);
+      __ vtbl(kScratchQuadReg.low(), table, kScratchQuadReg.low());
+      __ vtbl(kScratchQuadReg.high(), table, kScratchQuadReg.high());
+      __ vmov(dst, kScratchQuadReg);

[martyn.capewell, 2017/05/05 14:28:49]

If you know dst doesn't alias src0 or src1, vtbl can write into dst directly.

[bbudge, 2017/05/05 20:36:28]

Nice, done.

+      break;
+    }
     case kArmS8x16ZipLeft: {
       Simd128Register dst = i.OutputSimd128Register(),
                       src1 = i.InputSimd128Register(1);
       DCHECK(dst.is(i.InputSimd128Register(0)));
       // src0 = [0, 1, 2, 3, ... 15], src1 = [16, 17, 18, 19, ... 31]
       __ vmov(dst.high(), src1.low());
       __ vzip(Neon8, dst.low(), dst.high());  // dst = [0, 16, 1, 17, ... 23]
       break;
     }
     case kArmS8x16ZipRight: {
@@ skipping 39 matching lines @@
       // src0 = [16, 17, 18, 19, ... 31], src1 = [0, 1, 2, 3, ... 15] (flipped).
       __ vmov(kScratchQuadReg, src1);
       __ vtrn(Neon8, kScratchQuadReg, dst);  // dst = [1, 17, 3, 19, ... 31]
       break;
     }
     case kArmS8x16Concat: {
       __ vext(i.OutputSimd128Register(), i.InputSimd128Register(0),
               i.InputSimd128Register(1), i.InputInt4(2));
       break;
     }
+    case kArmS8x16Shuffle: {
+      Simd128Register dst = i.OutputSimd128Register(),
+                      src0 = i.InputSimd128Register(0),
+                      src1 = i.InputSimd128Register(1);
+      DwVfpRegister table_base = src0.low();
+      int table_size = GetVtblTableSize(src0, src1);
+      // The shuffle lane mask is a byte mask, materialize in kScratchQuadReg.
+      int scratch_s_base = kScratchQuadReg.code() * 4;
+      for (int j = 0; j < 4; j++) {
+        int32_t four_lanes = i.InputInt32(2 + j);
+        __ vmov(SwVfpRegister::from_code(scratch_s_base + j),
+                bit_cast<float>(four_lanes));
+      }
+      NeonListOperand table(table_base, table_size);
+      __ vtbl(kScratchQuadReg.low(), table, kScratchQuadReg.low());
+      __ vtbl(kScratchQuadReg.high(), table, kScratchQuadReg.high());
+      __ vmov(dst, kScratchQuadReg);
+      break;
+    }
     case kArmS32x2Reverse: {
       __ vrev64(Neon32, i.OutputSimd128Register(), i.InputSimd128Register(0));
       break;
     }
     case kArmS16x4Reverse: {
       __ vrev64(Neon16, i.OutputSimd128Register(), i.InputSimd128Register(0));
       break;
     }
     case kArmS16x2Reverse: {
       __ vrev32(Neon16, i.OutputSimd128Register(), i.InputSimd128Register(0));
@@ skipping 826 matching lines @@
       padding_size -= v8::internal::Assembler::kInstrSize;
     }
   }
 }
 
 #undef __
 
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8