Move validator_x86_XX.rl out of unreviewed.

src/trusted/validator_ragel/validator_x86_32.rl

 /*
  * Copyright (c) 2012 The Native Client Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * This is the core of ia32-mode validator.  Please note that this file
  * combines ragel machine description and C language actions.  Please read
  * validator_internals.html first to understand how the whole thing is built:
  * it explains how the byte sequences are constructed, what constructs like
  * "@{}" or "REX_WRX?" mean, etc.
  */
 
 #include <assert.h>
 #include <errno.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "native_client/src/trusted/validator_ragel/bitmap.h"
-#include "native_client/src/trusted/validator_ragel/unreviewed/validator_internal.h"
+#include "native_client/src/trusted/validator_ragel/validator_internal.h"
 
 /* Ignore this information: it's not used by security model in IA32 mode.  */
 #undef GET_VEX_PREFIX3
 #define GET_VEX_PREFIX3 0
 #undef SET_VEX_PREFIX3
 #define SET_VEX_PREFIX3(P)

[halyavin, 2013/03/19 13:44:44]

Check if these lines are necessary.

[khim, 2013/03/19 14:54:46]

Yes, they are.  Added TODO.

 
 %%{
   machine x86_32_validator;
   alphtype unsigned char;
   variable p current_position;
   variable pe end_of_bundle;
   variable eof end_of_bundle;
   variable cs current_state;
 
   include byte_machine "byte_machines.rl";
@@ skipping 15 matching lines @@
   include immediate_fields_parsing_ia32
     "native_client/src/trusted/validator_ragel/unreviewed/parse_instruction.rl";
   include relative_fields_validator_actions
     "native_client/src/trusted/validator_ragel/unreviewed/parse_instruction.rl";
   include relative_fields_parsing
     "native_client/src/trusted/validator_ragel/unreviewed/parse_instruction.rl";
   include cpuid_actions
     "native_client/src/trusted/validator_ragel/unreviewed/parse_instruction.rl";
 
   # Action which marks last byte as not immediate.  Most 3DNow! instructions,
-  # some AVX and XOP instructions have this property.   It's referenced by
-  # decode_x86_32 machine in [autogenerated] "validator_x86_32_instruction.rl"
-  # file.
+  # some AVX and XOP instructions have this property.
+  #
+  # This action is referenced by decode_x86_32 ragel machine in [autogenerated]
+  # "validator_x86_32_instruction.rl" file.
   action last_byte_is_not_immediate {

[halyavin, 2013/03/19 13:44:44]

Think on removing code duplication here.

[khim, 2013/03/19 14:54:46]

Done.

     instruction_info_collected |= LAST_BYTE_IS_NOT_IMMEDIATE;
   }
 
   include decode_x86_32 "validator_x86_32_instruction.rl";
 
   special_instruction =
    # and $~0x1f, %eXX  call  %eXX
    #                   vvvvvvvvvv
     (0x83 0xe0 0xe0    0xff (0xd0|0xe0)  | # naclcall/jmp %eax
      0x83 0xe1 0xe0    0xff (0xd1|0xe1)  | # naclcall/jmp %ecx
      0x83 0xe2 0xe0    0xff (0xd2|0xe2)  | # naclcall/jmp %edx
      0x83 0xe3 0xe0    0xff (0xd3|0xe3)  | # naclcall/jmp %ebx
      0x83 0xe4 0xe0    0xff (0xd4|0xe4)  | # naclcall/jmp %esp
      0x83 0xe5 0xe0    0xff (0xd5|0xe5)  | # naclcall/jmp %ebp
      0x83 0xe6 0xe0    0xff (0xd6|0xe6)  | # naclcall/jmp %esi
      0x83 0xe7 0xe0    0xff (0xd7|0xe7))   # naclcall/jmp %edi
    #                   ^^^^       ^^^^
    # and $~0x1f, %eXX     jmp %eXX
     @{
       UnmarkValidJumpTarget((current_position - data) - 1, valid_targets);
       instruction_begin -= 3;
       instruction_info_collected |= SPECIAL_INSTRUCTION;
     } |
     (0x65 0xa1 (0x00|0x04) 0x00 0x00 0x00      | # mov %gs:0x0/0x4,%eax
      0x65 0x8b (0x05|0x0d|0x015|0x1d|0x25|0x2d|0x35|0x3d)
           (0x00|0x04) 0x00 0x00 0x00);           # mov %gs:0x0/0x4,%reg
 
-  # Check if call is properly aligned
+  # Ragel machine which checks if call is properly aligned.

[halyavin, 2013/03/19 13:44:44]

Ragel machine that accepts one call instruction or call superinstruction and
checks if call is properly aligned.

[khim, 2013/03/19 14:54:46]

Done.

   #
   # For direct call we explicitly encode all variations.  For indirect call
   # we accept all the special instructions which ends with register-addressed
   # indirect call.
   call_alignment =
     ((one_instruction &
       # Direct call
       ((data16 0xe8 rel16) |
        (0xe8 rel32))) |

[halyavin, 2013/03/19 13:44:44]

direct_call = (data16 ...) 

[khim, 2013/03/19 14:54:46]

Done.

      (special_instruction &
       # Indirect call
       (any* data16? 0xff ((opcode_2 | opcode_3) any* &

[halyavin, 2013/03/19 13:44:44]

indirect_call_by_register = data16? 0xff (opcode_2 & modrm_registers)

[khim, 2013/03/19 14:54:46]

Done.

                           modrm_registers))))
     # Call instruction must aligned to the end of bundle.  Previously this was
     # strict requirement, today it's just warning to aid with debugging.
     @{
       if (((current_position - data) & kBundleMask) != kBundleMask)
         instruction_info_collected |= BAD_CALL_ALIGNMENT;
     };
 
   # This action calls user's callback (if needed) and cleans up validator's

[halyavin, 2013/03/19 13:44:44]

user's -> user, validator's -> internal validator state.

[khim, 2013/03/19 14:54:46]

Done.

   # internal state.
   #
   # We call the user callback if there are validation errors or if the

[halyavin, 2013/03/19 13:44:44]

We call the user callback either on validation errors or on every instruction,
depending on CALL_USER_CALLBACK_ON_EACH_INSTRUTION option.

[khim, 2013/03/19 14:54:46]

Done.

   # CALL_USER_CALLBACK_ON_EACH_INSTRUCTION option is used.
   #
   # After that we move instruction_begin and clean all the variables which
   # only used in the processing of a single instruction (prefixes, operand

[halyavin, 2013/03/19 13:44:44]

which are not used in superinstruction and sandboxing processing.

[khim, 2013/03/19 14:54:46]

I'm not sure that's clarification: I say that I clean up all the variables which
are used for one instruction only, you say that we clear all the variables
except for variables used in superinstruction and sandboxing processing which
plethora of questions.

   # states and instruction_info_collected).
   action end_of_instruction_cleanup {
     /* Mark start of this instruction as a valid target for jump.  */
     MarkValidJumpTarget(instruction_begin - data, valid_targets);
 
     /* Call user-supplied callback.  */
     instruction_end = current_position + 1;
     if ((instruction_info_collected & VALIDATION_ERRORS_MASK) ||
         (options & CALL_USER_CALLBACK_ON_EACH_INSTRUCTION)) {
       result &= user_callback(instruction_begin, instruction_end,
                               instruction_info_collected, callback_data);
     }
 
     /* On successful match the instruction_begin must point to the next byte
      * to be able to report the new offset as the start of instruction
      * causing error.  */

[halyavin, 2013/03/19 13:44:44]

We may set instruction_begin at the first byte of the instruction instead of
here but in the case of incorrect one byte instructions user callback may be
called before instruction_begin is set.

[khim, 2013/03/19 14:54:46]

Done.

     instruction_begin = instruction_end;
 
     /* Clear variables (well, one variable currently).  */
     instruction_info_collected = 0;
   }
 
   # This action reports fatal error detected by DFA.
   action report_fatal_error {
     result &= user_callback(instruction_begin, current_position,
                             UNRECOGNIZED_INSTRUCTION, callback_data);
     /*
      * Process the next bundle: "continue" here is for the "for" cycle in
      * the ValidateChunkIA32 function.
      *
      * It does not affect the case which we really care about (when code
      * is validatable), but makes it possible to detect more errors in one
      * run in tools like ncval.
      */
     continue;
   }
 
   # This is main ragel machine: it does 99% of validation work. There are only
-  # one thing to do if this machine accepts the bundles - check that direct
-  # jumps are correct. This is done in the following way:
+  # one thing to do if this ragel machine accepts the bundles - check that
+  # direct jumps are correct. This is done in the following way:
   #  * DFA fills two arrays: valid_targets and jump_dests.
   #  * ProcessInvalidJumpTargets checks that "jump_dests & !valid_targets == 0".
   # All other checks are done here.
   main := ((call_alignment | one_instruction | special_instruction)
      @end_of_instruction_cleanup)*
     $!report_fatal_error;
 
 }%%
 
+/*
+ * The "write data" statement causes Ragel to emit the constant static data
+ * needed by the ragel machine.
+ */
 %% write data;
 
-
 Bool ValidateChunkIA32(const uint8_t *data, size_t size,
                        uint32_t options,
                        const NaClCPUFeaturesX86 *cpu_features,
                        ValidationCallbackFunc user_callback,
                        void *callback_data) {
   bitmap_word valid_targets_small;
   bitmap_word jump_dests_small;
   bitmap_word *valid_targets;
   bitmap_word *jump_dests;
   const uint8_t *current_position;
@@ skipping 16 matching lines @@
       free(jump_dests);
       free(valid_targets);
       errno = ENOMEM;
       return FALSE;
     }
   }
 
   /*
    * This option is usually used in tests: we will process the whole chunk
    * in one pass. Usually each bundle is processed separately which means
-   * instructions (and super-instructions) can not cross borders of the bundle.
+   * instructions (and "superinstructions") can not cross borders of the bundle.
    */
   if (options & PROCESS_CHUNK_AS_A_CONTIGUOUS_STREAM)
     end_of_bundle = data + size;
   else
     end_of_bundle = data + kBundleSize;
 
   /*
    * Main loop.  Here we process the data array bundle-after-bundle.
    * Ragel-produced DFA does all the checks with one exception: direct jumps.
    * It collects the two arrays: valid_targets and jump_dests which are used
    * to test direct jumps later.
    */
   for (current_position = data;
        current_position < data + size;
        current_position = end_of_bundle,
        end_of_bundle = current_position + kBundleSize) {
     /* Start of the instruction being processed.  */
     const uint8_t *instruction_begin = current_position;
     /* Only used locally in the end_of_instruction_cleanup action.  */
     const uint8_t *instruction_end;
     uint32_t instruction_info_collected = 0;
     int current_state;
 
+    /*
+     * The "write init" statement causes Ragel to emit initialization code.
+     * This should be executed once before the ragel machine is started.
+     */
     %% write init;
+    /*
+     * The "write exec" statement causes Ragel to emit the ragel machine's
+     * execution code.
+     */
     %% write exec;
   }
 
   /*
    * Check the direct jumps.  All the targets from jump_dests must be in
    * valid_targets.
    */
   result &= ProcessInvalidJumpTargets(data, size, valid_targets, jump_dests,
                                       user_callback, callback_data);
 
   /* We only use malloc for a large code sequences  */
   if (jump_dests != &jump_dests_small) free(jump_dests);
   if (valid_targets != &valid_targets_small) free(valid_targets);
   if (!result) errno = EINVAL;
   return result;
 }