Enable vdiff test on validator bots

src/trusted/validator/x86/testing/enuminsts/vdiff.c

 /*
  * Copyright (c) 2012 The Native Client Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /* vdiff.c
  * exhaustive instruction enumeration test for x86 Native Client validators.
  *
  * This file is based on enuminsts.c, but specialized to comparing two
  * validators instead of decoders. The enuminsts.c implementation also
  * had a bunch of Xed-specific logic which complicated the validator
  * comparison in unhelpful ways.
  */
 
 #ifndef NACL_TRUSTED_BUT_NOT_TCB
 #error("This file is not meant for use in the TCB.")
 #endif
+#if NACL_WINDOWS
+#define _CRT_RAND_S  /* enable decl of rand_s() */
+#endif
 
 #include "native_client/src/trusted/validator/x86/testing/enuminsts/enuminsts.h"
 
 #include <ctype.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <time.h>
 
 #include "native_client/src/include/portability_io.h"
 #include "native_client/src/shared/platform/nacl_log.h"
 #include "native_client/src/shared/utils/flags.h"
 #include "native_client/src/trusted/validator/x86/testing/enuminsts/str_utils.h"
 #include "native_client/src/trusted/validator/x86/testing/enuminsts/text2hex.h"
 
 /* Defines the maximum buffer size used to hold text generated for the
  * disassembly of instructions, and corresponding error messages.
  */
 #define kBufferSize 1024
 
 /* When true, print more messages (i.e. verbosely). */
 static Bool gVerbose = FALSE;
 
 /* When true, don't print out messages. That is, only print instructions
  * defined by --print directives.
  */
 static Bool gSilent = FALSE;
 
 /* When true, don't report consecutive errors for consecutive instructions
  * with the same instruction mnemonic.
  */
 static Bool gSkipRepeatReports = FALSE;
 
+/* Set to true to enable checking of mnemonics (opcode names).
+ */
+static Bool gCheckMnemonics = TRUE;
+
 /* Count of errors that have a high certainty of being exploitable. */
 static int gSawLethalError = 0;
 
 /* Defines the assumed text address for the test instruction */
 const int kTextAddress = 0x1000;
 
 /* If non-negative, defines the prefix to test. */
 static unsigned int gPrefix = 0;
 
 /* If non-negative, defines the opcode to test. */
 static int gOpcode = -1;
 
+/* This option triggers a set of behaviors that help produce repeatable
+ * output, for easier diffs on the buildbots.
+ */
+static Bool gEasyDiffMode;
+
 /* The production and new R-DFA validators */
 NaClEnumeratorDecoder* vProd;
 NaClEnumeratorDecoder* vDFA;
 
 /* The name of the executable (i.e. argv[0] from the command line). */
-static char *gArgv0 = "argv0";
+static const char *gArgv0 = "argv0";
+#define FLAG_EasyDiff "--easydiff"
 
 /* Records that unexpected internal error occurred. */
 void InternalError(const char *why) {
   fprintf(stderr, "%s: Internal Error: %s\n", gArgv0, why);
   gSawLethalError = 1;
 }
 
 /* Records that a fatal (i.e. non-recoverable) error occurred. */
 void ReportFatalError(const char* why) {
   char buffer[kBufferSize];
@@ skipping 58 matching lines @@
 }
 
 static void CheckMnemonics(NaClEnumerator* pinst, NaClEnumerator* dinst) {
   const char* prod_opcode = vProd->_get_inst_mnemonic_fn(pinst);
   const char* dfa_opcode = vDFA->_get_inst_mnemonic_fn(dinst);
 
   if (0 != strcmp(prod_opcode, dfa_opcode)) {
     /* avoid redundant messages... */
     if (NotOpcodeRepeat(prod_opcode)) {
       printf("Warning: OPCODE MISMATCH: %s != %s\n", prod_opcode, dfa_opcode);
-      /* PrintDisassembledInstructionVariants(pinst, dinst); */
     }
   }
 }
 
 struct vdiff_stats {
   int64_t tried;
   int64_t valid;
   int64_t invalid;
   int64_t errors;
 } gVDiffStats = {0, 0, 0, 0};
 
 static void IncrTried() {
   gVDiffStats.tried += 1;
 }
 
 static void IncrValid() {
   gVDiffStats.valid += 1;
 }
 
 static void IncrInvalid() {
   gVDiffStats.invalid += 1;
 }
 
 static void IncrErrors() {
   gVDiffStats.errors += 1;
 }
 
 static void PrintStats() {
-  printf("valid: %" NACL_PRIu64 "\n", gVDiffStats.valid);
-  printf("invalid: %" NACL_PRIu64 "\n", gVDiffStats.invalid);
+  if (!gEasyDiffMode) {
+    printf("valid: %" NACL_PRIu64 "\n", gVDiffStats.valid);
+    printf("invalid: %" NACL_PRIu64 "\n", gVDiffStats.invalid);
+  }
   printf("errors: %" NACL_PRIu64 "\n", gVDiffStats.errors);
   printf("tried: %" NACL_PRIu64 "\n", gVDiffStats.tried);
   printf("    =? %" NACL_PRIu64 " valid + invalid + errors\n",
          gVDiffStats.valid + gVDiffStats.invalid + gVDiffStats.errors);
 }
 
 static void InitInst(NaClEnumerator *nacle,
                      uint8_t *itext, size_t nbytes)
 {
   memcpy(nacle->_itext, itext, nbytes);
@@ skipping 24 matching lines @@
     rdfa_okay = vDFA->_maybe_inst_validates_fn(&dinst);
 
     if (prod_okay) {
       if (rdfa_okay) {
         if (vProd->_inst_length_fn(&pinst) ==
             vDFA->_inst_length_fn(&dinst)) {
           /* Both validators see a legal instruction, */
           /* and they agree on critical details.      */
           IncrValid();
           /* Warn if decoders disagree opcode name. */
-          CheckMnemonics(&pinst, &dinst);
+          if (gCheckMnemonics) CheckMnemonics(&pinst, &dinst);
         } else {
           DecoderError("LENGTH MISMATCH", &pinst, &dinst, "");
           IncrErrors();
         }
       } else {
         /* Validators disagree on instruction legality */
         DecoderError("VALIDATORS DISAGREE", &pinst, &dinst, "");
         IncrErrors();
       }
     } else if (rdfa_okay) {
       /* Validators disagree on instruction legality */
       DecoderError("VALIDATORS DISAGREE", &pinst, &dinst, "");
       IncrErrors();
     } else {
       /* Both validators see an illegal instruction */
       IncrInvalid();
     }
 
-    /* no error */
     if (gVerbose) {
       PrintDisassembledInstructionVariants(&pinst, &dinst);
     }
   } while (0);
 }
 
+/* A function type for instruction "TestAll" functions.
+ * Parameters:
+ *     prefix: up to four bytes of prefix.
+ *     prefix_length: size_t on [0..4] specifying length of prefix.
+ *     print_prefix: For easy diff of test output, avoid printing
+ *         the value of a randomly selected REX prefix.
+ */
+typedef void (*TestAllFunction)(const unsigned int prefix,
+                                const size_t prefix_length,
+                                const char* print_prefix);
+
+/* Create a char* rendition of a prefix string, appending bytes
+ * in ps. This is used to create output that is easy to diff when
+ * a random REX byte is used.
+ */
+static char* StrPrefix(const unsigned int prefix, char* ps, char* str) {

[Karl, 2012/08/28 21:23:52]

It is not clear to me what the arguments here are. A comment would be helpful.

[Brad Chen, 2012/08/28 22:09:42]

Done.

+  sprintf(str, "%x%s", prefix, (ps == NULL) ? "" : ps);
+  return str;
+}
+
 /* Enumerate and test all 24-bit opcode+modrm+sib patterns for a
  * particular prefix.
  */
-static void TestAllWithPrefix(unsigned int prefix, size_t prefix_length) {
-  const int kInstByteCount = NACL_ENUM_MAX_INSTRUCTION_BYTES;
-  const int kIterByteCount = 3;
+static void TestAllWithPrefix(const unsigned int prefix,
+                              const size_t prefix_length,
+                              const char* print_prefix) {
+  const size_t kInstByteCount = NACL_ENUM_MAX_INSTRUCTION_BYTES;
+  const size_t kIterByteCount = 3;
   InstByteArray itext;
   size_t i;
   int op, modrm, sib;
   int min_op;
   int max_op;
 
   if ((gPrefix > 0) && (gPrefix != prefix)) return;
 
-  PrintProgress("TestAllWithPrefix(%x)\n", prefix);
+  PrintProgress("TestAllWithPrefix(%s)\n", print_prefix);
   /* set up prefix */
   memcpy(itext, &prefix, prefix_length);
   /* set up filler bytes */
   for (i = prefix_length + kIterByteCount;
        i < NACL_ENUM_MAX_INSTRUCTION_BYTES; i++) {
     itext[i] = (uint8_t)i;
   }
   if (gOpcode < 0) {
     min_op = 0;
     max_op = 256;
   } else {
     min_op = gOpcode;
     max_op = gOpcode + 1;
   }
   for (op = min_op; op < max_op; op++) {
     itext[prefix_length] = op;
     PrintProgress("%02x 00 00\n", op);
     for (modrm = 0; modrm < 256; modrm++) {
       itext[prefix_length + 1] = modrm;
       for (sib = 0; sib < 256; sib++) {
         itext[prefix_length + 2] = sib;
         TryOneInstruction(itext, kInstByteCount);
       }
     }
   }
 }
 
+#if NACL_TARGET_SUBARCH == 64
+/* REX prefixes range from 0x40 to 0x4f. */
+const uint32_t kREXBase = 0x40;
+const uint32_t kREXRange = 0x10;
+const uint32_t kREXMax = 0x50;  /* kREXBase + kREXRange */

[Karl, 2012/08/28 21:23:52]

Add a blank line here?

[Brad Chen, 2012/08/28 22:09:42]

Done.

+/* Generate a random REX prefix, to use for the entire run. */
+static uint32_t RandomRexPrefix() {
+  static uint32_t static_rex_prefix = 0;
+
+  if (0 == static_rex_prefix) {
+#if NACL_LINUX || NACL_OSX
+    static_rex_prefix = kREXBase + (random() % kREXRange);
+#elif NACL_WINDOWS
+    if (rand_s(&static_rex_prefix) != 0) {
+      ReportFatalError("rand_s() failed\n");
+    } else {
+      static_rex_prefix = kREXBase + (static_rex_prefix % kREXRange);
+    }
+#else
+# error "Unknown operating system."
+#endif
+  }
+  return static_rex_prefix;
+}
+#endif
+
 /* For x86-64, enhance the iteration by looping through REX prefixes.
  */
-static void TestAllWithPrefixREX(unsigned int prefix, size_t prefix_length) {
+static void WithREX(TestAllFunction testall,
+                    const unsigned int prefix,
+                    const size_t prefix_length) {
+  char pstr[kBufferSize];
 #if NACL_TARGET_SUBARCH == 64
-  unsigned char REXp;
+  unsigned char irex;
   unsigned int rprefix;
   /* test with REX prefixes */
-  for (REXp = 0x40; REXp < 0x50; REXp++) {
-    rprefix = (prefix << 8 | REXp);
-    printf("Testing with prefix %x\n", rprefix);
-    TestAllWithPrefix(rprefix, prefix_length + 1);
+  printf("WithREX(testall, %x, %d, %d)\n", prefix,
+         (int)prefix_length, gEasyDiffMode);
+  if (gEasyDiffMode) {
+    printf("With random REX prefix.\n");
+    irex = RandomRexPrefix();
+    rprefix = (prefix << 8 | irex);

[Karl, 2012/08/28 21:23:52]

Could this operation be converted to a MACRO. It appears several times in the
code, and isn't obvious that you are appending a value to the prefix.

[Brad Chen, 2012/08/28 22:09:42]

Done.

+    testall(rprefix, prefix_length + 1, StrPrefix(prefix, "XX", pstr));
+  } else {
+    for (irex = kREXBase; irex < kREXMax; irex++) {
+      rprefix = (prefix << 8 | irex);
+      printf("With REX prefix %x\n", rprefix);
+      testall(rprefix, prefix_length + 1, StrPrefix(rprefix, "", pstr));
+    }
   }
 #endif
   /* test with no REX prefix */
-  TestAllWithPrefix(prefix, prefix_length);
+  testall(prefix, prefix_length, StrPrefix(prefix, NULL, pstr));
 }
 
 /* For all prefixes, call TestAllWithPrefix() to enumrate and test
  * all instructions.
  */
 static void TestAllInstructions() {
   gSkipRepeatReports = TRUE;
   /* NOTE: Prefix byte order needs to be reversed when written as
    * an integer. For example, for integer prefix 0x3a0f, 0f will
    * go in instruction byte 0, and 3a in byte 1.
    */
-  /* TODO(bradchen): extend enuminsts-64 to iterate over 64-bit prefixes. */
-  TestAllWithPrefixREX(0, 0);
-  TestAllWithPrefixREX(0x0f, 1);
-  TestAllWithPrefixREX(0x0ff2, 2);
-  TestAllWithPrefixREX(0x0ff3, 2);
-  TestAllWithPrefixREX(0x0f66, 2);
-  TestAllWithPrefixREX(0x0f0f, 2);
-  TestAllWithPrefixREX(0x380f, 2);
-  TestAllWithPrefixREX(0x3a0f, 2);
-  TestAllWithPrefixREX(0x380f66, 3);
-  TestAllWithPrefixREX(0x380ff2, 3);
-  TestAllWithPrefixREX(0x3a0f66, 3);
+  WithREX(TestAllWithPrefix, 0, 0);
+  WithREX(TestAllWithPrefix, 0x0f, 1);     /* two-byte opcode */
+  WithREX(TestAllWithPrefix, 0x0ff2, 2);   /* SSE2  */
+  WithREX(TestAllWithPrefix, 0x0ff3, 2);   /* SSE   */
+  WithREX(TestAllWithPrefix, 0x0f66, 2);   /* SSE2  */
+  WithREX(TestAllWithPrefix, 0x380f, 2);   /* SSSE3 */
+  WithREX(TestAllWithPrefix, 0x3a0f, 2);   /* SSE4  */
+  WithREX(TestAllWithPrefix, 0x380f66, 3); /* SSE4+ */
+  WithREX(TestAllWithPrefix, 0x380ff2, 3); /* SSE4+ */
+  WithREX(TestAllWithPrefix, 0x3a0f66, 3); /* SSE4+ */
 }
 
 /* Used to test one instruction at a time, for example, in regression
  * testing, or for instruction arguments from the command line.
  */
-static void TestOneInstruction(char *asciihex) {
+static void TestOneInstruction(const char *asciihex) {
   InstByteArray ibytes;
   int nbytes;
 
   nbytes = Text2Bytes(ibytes, asciihex, "Command-line argument", -1);
   if (nbytes == 0) return;
   if (gVerbose) {
     int i;
     printf("trying %s (", asciihex);
     for (i = 0; i < nbytes; ++i) {
       printf("%02x", ibytes[i]);
@@ skipping 28 matching lines @@
    */
   TestOneInstruction("262e7e00");
   TestOneInstruction("2e3e7900");
   /* From the AMD manual, "An instruction may have only one REX prefix */
   /* which must immediately precede the opcode or first excape byte    */
   /* in the instruction encoding." */
   TestOneInstruction("406601d8");  /* illegal; REX before data16 */
   TestOneInstruction("664001d8");  /* legal; REX after data16    */
   TestOneInstruction("414001d8");  /* illegal; two REX bytes     */
 
-  /* Reset the opcode repeat test, so as not to silence erros */
+  /* Reset the opcode repeat test, so as not to silence errors */
   /* that happened in the regression suite. */
   (void)NotOpcodeRepeat("");
 }
 
 /* Define decoders that can be registered. */
 extern NaClEnumeratorDecoder* RegisterNaClDecoder();
 extern NaClEnumeratorDecoder* RegisterRagelDecoder();
 
 /* Initialize the set of available decoders. */
-static void NaClInitializeAvailableDecoders() {
+static void VDiffInitializeAvailableDecoders() {
   vProd = RegisterNaClDecoder();
   vDFA = RegisterRagelDecoder();
 }
 
-int main(int argc, char *argv[]) {
-  int testargs;
+static int ParseArgv(const int argc, const char* argv[]) {
+  int nextarg;
+
+  gArgv0 = argv[0];
+  nextarg = 1;
+  if (nextarg < argc &&
+      0 == strcmp(argv[nextarg], FLAG_EasyDiff)) {
+    gEasyDiffMode = TRUE;
+    gCheckMnemonics = FALSE;
+    nextarg += 1;
+  }
+  return nextarg;
+}
+
+int main(const int argc, const char *argv[]) {
+  int nextarg;
 
   NaClLogModuleInit();
   NaClLogSetVerbosity(LOG_FATAL);
-  NaClInitializeAvailableDecoders();
+#if NACL_LINUX || NACL_OSX
+  srandom(time(NULL));
+#endif
+  VDiffInitializeAvailableDecoders();
 
-  gArgv0 = argv[0];
-  testargs = 1;
-
-  if (testargs == argc) {
+  nextarg = ParseArgv(argc, argv);
+  if (nextarg == argc) {
     if (gPrefix == 0) RunRegressionTests();
     TestAllInstructions();
   } else {
     int i;
     gVerbose = TRUE;
-    for (i = testargs; i < argc; ++i) {
+    for (i = nextarg; i < argc; ++i) {
       TestOneInstruction(argv[i]);
     }
   }
   PrintStats();
 }