Add support for more AVX instructions.

syzygy/core/disassembler_util.cc

 // Copyright 2012 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
 #include "syzygy/core/disassembler_util.h"
 
+#include <algorithm>
+
 #include "base/logging.h"
 #include "base/strings/stringprintf.h"
 #include "mnemonics.h"  // NOLINT
 
 namespace core {
 
 namespace {
 
 // Return the size of a 3-byte VEX encoded instruction.
 //
@@ skipping 15 matching lines @@
 //
 // |map_select| Indicates the opcode map that should be used for this
 // instruction.
 //
 // See http://wiki.osdev.org/X86-64_Instruction_Encoding#Three_byte_VEX_escape_prefix
 // for more details.
 size_t Get3ByteVexEncodedInstructionSize(_CodeInfo* ci) {
   DCHECK_EQ(0xC4, ci->code[0]);
   // Switch case based on the opcode map used by this instruction.
   switch (ci->code[1] & 0x1F) {
-    case 0x01: {
-      switch (ci->code[3]) {
-        case 0x1D: return 5;  // vpermd
-        default: break;
-      }
-      break;
-    }
     case 0x02: {
       switch (ci->code[3]) {
         case 0x13: return 5;  // vcvtps2ps
         case 0x18: return 5;  // vbroadcastss
         case 0x36: return 5;  // vpermd
+        case 0x58: return 6;  // vpbroadcastd
         case 0x5A: return 6;  // vbroadcasti128
         case 0x78: return 5;  // vpbroadcastb
-        default: break;
+        case 0x8C: return 5;  // vpmaskmovd
+        case 0x8E: return 5;  // vpmaskmovd
+        case 0x90: return 6;  // vpgatherdd
+        default:
+          break;
       }
       break;
     }
     case 0x03: {
       switch (ci->code[3]) {
         case 0x00: return 6;  // vpermq
         case 0x1D: return 6;  // vcvtps2ph
         case 0x38: return 7;  // vinserti128
         case 0x39: return 6;  // vextracti128
         default: break;
       }
       break;
     }
     default:
       break;
   }
+
+  // Print the instructions that we haven't been able to decompose in a format
+  // that can easily be pasted into ODA (https://onlinedisassembler.com/).
+  const int kMaxBytes = 10;
+  size_t byte_count = std::min(ci->codeLen, kMaxBytes);
+  std::string instruction_bytes;
+  for (size_t i = 0; i < byte_count; ++i) {
+    base::StringAppendF(&instruction_bytes, "%02X", ci->code[i]);
+    if (i != byte_count - 1)
+      instruction_bytes += " ";
+  }
+  if (ci->codeLen > kMaxBytes)
+    instruction_bytes += "...";
+  LOG(WARNING) << "Failed to decompose a VEX encoded instructions with the "
+               << "following bytes: " << instruction_bytes;
   return 0;
 }
 
 void AdjustOperandSizeTo16Bit(_Operand* op) {
   DCHECK_EQ(32, op->size);
 
   op->size = 16;
   if (op->type == O_REG) {
     DCHECK(op->index >= R_EAX && op->index < R_AX);
     // Size classes for registers are 16 indices apart.
@@ skipping 336 matching lines @@
 
     default: return assm::kRegisterNone;
   }
 }
 
 const Register& GetRegister(uint32_t distorm_reg_type) {
   return Register::Get(GetRegisterId(distorm_reg_type));
 }
 
 }  // namespace core