Improve the decoding of the VEX encoded instructions.

syzygy/core/disassembler_util_unittest_vex_utils.h

+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SYZYGY_CORE_DISASSEMBLER_UTIL_UNITTEST_VEX_UTILS_H_
+#define SYZYGY_CORE_DISASSEMBLER_UTIL_UNITTEST_VEX_UTILS_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+namespace core {
+namespace unittests {
+
+// Examples of 3-byte VEX instructions that aren't properly handled by Distorm.
+extern const std::vector<std::vector<uint8_t>> kVexInstructions;
+
+// Most of the 3-byte VEX instructions end up with a Mod R/M encoded operand
+// and some optional extra bytes (depending on the value of the Mod R/M byte),
+// see the comment of core::GetOperandBytesSize.
+//
+// This is a list of all the 3-byte VEX instruction that we support and all the
+// possible variations of the Mod R/M byte, all these instructions have been
+// verified in a disassembler.
+extern const std::vector<std::vector<uint8_t>> kVexInstructionsModRMVariants;
+
+}  // namespace unittests
+}  // namespace core
+
+#endif  // SYZYGY_CORE_DISASSEMBLER_UTIL_UNITTEST_VEX_UTILS_H_