Improve the decoding of the VEX encoded instructions.

syzygy/core/disassembler_util_unittest.cc

 // Copyright 2012 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
 #include "syzygy/core/disassembler_util.h"
 
+#include <vector>
+
 #include "base/logging.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include "syzygy/assm/unittest_util.h"
+#include "syzygy/core/disassembler_util_unittest_vex_utils.h"
 
 namespace core {
 
 namespace {
 
 // Decompose a block of code using distorm wrapper.
 _DecodeResult DecomposeCode(const uint8_t* code_data,
                             size_t length,
                             _DInst result[],
                             const unsigned int max_results,
@@ skipping 80 matching lines @@
 const uint8_t kRet[] = {0xC3};
 const uint8_t kRetN[] = {0xC2, 0x08, 0x00};
 const uint8_t kJe[] = {0x74, 0xCA};
 const uint8_t kSysEnter[] = {0x0F, 0x34};
 const uint8_t kSysExit[] = {0x0F, 0x35};
 
 // Interrupts.
 const uint8_t kInt2[] = {0xCD, 0x02};
 const uint8_t kInt3[] = {0xCC};
 
-// VEX encoded instructions that Distorm doesn't handle properly.
-const std::vector<std::vector<uint8_t>> kVexInstructions = {
-    // AVX instructions.
-    {0xC4, 0xE3, 0xFD, 0x00, 0xED, 0x44},        // vpermq
-    {0xC4, 0xE2, 0x4D, 0x36, 0xC0},              // vpermd
-    {0xC4, 0xE2, 0x7D, 0x5A, 0x45, 0xD0},        // vbroadcasti128
-    {0xC4, 0xE3, 0x7D, 0x38, 0x2C, 0x0F, 0x01},  // vinserti128
-    {0xC4, 0xE2, 0x79, 0x78, 0xC0},              // vpbroadcastb
-    {0xC4, 0xE2, 0x7D, 0x58, 0x40, 0x04},        // vpbroadcastd
-    {0xC4, 0xE2, 0x7D, 0x18, 0xC0},              // vbroadcastss
-    {0xC4, 0xE3, 0x7D, 0x39, 0xC8, 0x01},        // vextracti128
-    {0xC4, 0xE2, 0x7D, 0x90, 0x1C, 0x88},        // vpgatherdd
-    {0xC4, 0xE2, 0x7D, 0x8C, 0x00},              // vpmaskmovd
-    {0xC4, 0xE2, 0x7D, 0x8E, 0x90},              // vpmaskmovd
-
-    // F16C instructions.
-    {0xC4, 0xE3, 0x79, 0x1D, 0xC8, 0x00},  // vcvtps2ph
-    {0xC4, 0xE2, 0x79, 0x13, 0xE0},        // vcvtps2ps
-};
 
 void TestBadlyDecodedInstruction(const uint8_t* code, size_t code_length) {
   _DInst inst[1] = {};
   unsigned int inst_count = 0;
   _DecodeResult result = RawDecomposeCode(
       code, code_length, inst, arraysize(inst), &inst_count);
   EXPECT_EQ(DECRES_MEMORYERR, result);
   EXPECT_EQ(0u, inst_count);
 
   result = DecomposeCode(
@@ skipping 214 matching lines @@
                           results,
                           kMaxResults,
                           &result_count));
   EXPECT_EQ(1, result_count);
   EXPECT_EQ(64, results[0].ops[0].size);
 }
 
 // If this test starts failing then Distorm now properly handles the AVX2
 // instructions. Please remove the workaround in disassembler_util.cc.
 TEST(DisassemblerUtilTest, TestBadlyDecodedVexInstructions) {
-  for (const auto iter : kVexInstructions) {
+  for (const auto iter : unittests::kVexInstructions) {
     EXPECT_NO_FATAL_FAILURE(
         TestBadlyDecodedInstruction(iter.data(), iter.size()));
   }
 }
 
+TEST(DisassemblerUtilTest, TestBadlyDecodedVexInstructionsModRMVariants) {
+  for (const auto iter : unittests::kVexInstructionsModRMVariants) {
+    _DInst inst[1] = {};
+    unsigned int inst_count = 0;
+
+    _DecodeResult result = DecomposeCode(iter.data(), iter.size(), inst,
+                                         arraysize(inst), &inst_count);
+    EXPECT_EQ(DECRES_SUCCESS, result);
+    EXPECT_EQ(1u, inst_count);
+    EXPECT_EQ(iter.size(), inst[0].size);
+  }
+}
+
 TEST(DisassemblerUtilTest, TestBadlyDecodedCRC32) {
   // CRC32 with a 16 bit operand size prefix is not handled correctly by
   // distorm.
   EXPECT_NO_FATAL_FAILURE(
       TestBadlyDecodedInstruction(kCrc32CX, sizeof(kCrc32CX)));
 }
 
 }  // namespace core