Speed up x86-64 validator by inlining heavily called routines. Speeds up

src/trusted/validator/x86/ncval_reg_sfi/nc_jumps.c

 /*
  * Copyright (c) 2011 The Native Client Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 /*
  * nc_jumps.c - Validate where valid jumps can occur.
  */
 
 #include <assert.h>
 #include <stdlib.h>
 
 #include "native_client/src/trusted/validator/x86/ncval_reg_sfi/nc_jumps.h"
 
 #include "native_client/src/shared/platform/nacl_log.h"
-#include "native_client/src/trusted/validator/x86/decoder/ncop_exps.h"
-#include "native_client/src/trusted/validator/x86/decoder/nc_inst_iter.h"
 #include "native_client/src/trusted/validator/x86/decoder/nc_inst_state_internal.h"
 #include "native_client/src/trusted/validator/x86/decoder/nc_inst_trans.h"
 #include "native_client/src/trusted/validator/x86/ncval_reg_sfi/address_sets.h"
 #include "native_client/src/trusted/validator/x86/ncval_reg_sfi/ncvalidate_iter.h"
 #include "native_client/src/trusted/validator/x86/ncval_reg_sfi/ncvalidate_iter_internal.h"
 
 /* To turn on debugging of instruction decoding, change value of
  * DEBUGGING to 1.
  */
 #define DEBUGGING 0
 
 #include "native_client/src/shared/utils/debugging.h"
 
+#include "native_client/src/trusted/validator/x86/decoder/ncop_exps_inl.c"
+#include "native_client/src/trusted/validator/x86/decoder/nc_inst_iter_inl.c"
+#include "native_client/src/trusted/validator/x86/ncval_reg_sfi/address_sets_inl.c"
+
 Bool NACL_FLAGS_identity_mask = FALSE;
 
 static INLINE uint8_t NaClGetJumpMask(NaClValidatorState* state) {
   return NACL_FLAGS_identity_mask
       ? (uint8_t) 0xFF
       : (uint8_t) (~state->alignment_mask);
 }
 
 /* Generates a jump validator. */
 NaClJumpSets* NaClJumpValidatorCreate(NaClValidatorState* state) {
@@ skipping 39 matching lines @@
    * Otherwise, only allow if 0 mod 32.
    */
   DEBUG(NaClLog(LOG_INFO, "Add jump to jump sets: %"
                 NACL_PRIxNaClPcAddress" -> %"NACL_PRIxNaClPcAddress"\n",
                 from_address, to_address));
   if (state->vbase <= to_address && to_address < state->vlimit) {
     /* Remember address for checking later. */
     DEBUG(NaClLog(LOG_INFO, "Add jump to target: %"NACL_PRIxNaClPcAddress
                   " -> %"NACL_PRIxNaClPcAddress"\n",
                   from_address, to_address));
-    NaClAddressSetAdd(jump_sets->actual_targets, to_address, state);
+    NaClAddressSetAddInline(jump_sets->actual_targets, to_address, state);
   }
   /* The range check may not be strictly necessary given that we have
    * guard regions around the sandbox address space, but it shouldn't
    * hurt to disallow branches that overflow or underflow the address
    * space.
    */
   else if ((to_address & state->alignment_mask) == 0 &&
            (to_address & ~(NaClPcAddress) 0xffffffff) == 0) {
     /* Allow bundle-aligned jump. */
   }
@@ skipping 93 matching lines @@
                                       NaClJumpSets* jump_sets) {
   uint8_t mask;
   NaClInstState* and_state;
   NaClInstState* middle_state;
   const NaClInst* and_inst;
   const NaClInst* middle_inst;
   int op_1, op_2;
   NaClOpKind and_reg, and_64_reg, jump_reg, middle_reg;
   NaClExpVector* nodes;
   NaClExp* node;
-  jump_reg = NaClGetExpRegister(reg);
+  jump_reg = NaClGetExpRegisterInline(reg);
   DEBUG(NaClLog(LOG_INFO, "checking indirect jump: ");
         NaClInstStateInstPrint(NaClLogGetGio(), inst);
         gprintf(NaClLogGetGio(), "jump_reg = %s\n", NaClOpKindName(jump_reg)));
 
   /* Do the following block exactly once. Use loop so that "break" can
    * be used for premature exit of block.
    */
   do {
     /* Check and in 3 instruction sequence. */
-    if (!NaClInstIterHasLookbackState(iter, 2)) break;
-    and_state = NaClInstIterGetLookbackState(iter, 2);
+    if (!NaClInstIterHasLookbackStateInline(iter, 2)) break;
+    and_state = NaClInstIterGetLookbackStateInline(iter, 2);
     DEBUG(NaClLog(LOG_INFO, "and?: ");
           NaClInstStateInstPrint(NaClLogGetGio(), and_state));
     and_inst = NaClInstStateInst(and_state);
     if (((and_state->num_opcode_bytes == 0) ||
          (0x83 != and_state->bytes.byte[and_state->num_prefix_bytes])) ||
         InstAnd != and_inst->name) break;
     DEBUG(NaClLog(LOG_INFO, "and instruction\n"));
 
     /* Extract the values of the two operands for the and. */
     if (!NaClExtractBinaryOperandIndices(and_state, &op_1, &op_2)) break;
     DEBUG(NaClLog(LOG_INFO, "binary and\n"));
 
     /* Extract the destination register of the and. */
     nodes = NaClInstStateExpVector(and_state);
     node = &nodes->node[op_1];
     if (ExprRegister != node->kind) break;
-    and_reg = NaClGetExpRegister(node);
+    and_reg = NaClGetExpRegisterInline(node);
     DEBUG(NaClLog(LOG_INFO, "and_reg = %s\n", NaClOpKindName(and_reg)));
     and_64_reg = NaClGet64For32BitReg(and_reg);
     DEBUG(NaClLog(LOG_INFO, "and_64_reg = %s\n", NaClOpKindName(and_64_reg)));
     if (RegUnknown == and_64_reg) break;
     DEBUG(NaClLog(LOG_INFO, "registers match!\n"));
 
     /* Check that the mask is ok. */
     mask = NaClGetJumpMask(state);
     DEBUG(NaClLog(LOG_INFO, "mask = %"NACL_PRIx8"\n", mask));
     assert(0 != mask);  /* alignment must be either 16 or 32. */
     node = &nodes->node[op_2];
     if (ExprConstant != node->kind || mask != node->value) break;
     DEBUG(NaClLog(LOG_INFO, "is mask constant\n"));
 
     /* Check middle (i.e. lea/add) instruction in 3 instruction sequence. */
-    middle_state = NaClInstIterGetLookbackState(iter, 1);
+    middle_state = NaClInstIterGetLookbackStateInline(iter, 1);
     DEBUG(NaClLog(LOG_INFO, "middle inst: ");
           NaClInstStateInstPrint(NaClLogGetGio(), middle_state));
     middle_inst = NaClInstStateInst(middle_state);
 
     /* Extract the values of the two operands for the lea/add instruction. */
     if (!NaClExtractBinaryOperandIndices(middle_state, &op_1, &op_2)) break;
     DEBUG(NaClLog(LOG_INFO, "middle is binary, op_1 index = %d\n", op_1));
 
     /* Extract the destination register of the lea/and, and verify that
      * it is a register.
      */
     nodes = NaClInstStateExpVector(middle_state);
     node = &nodes->node[op_1];
     if (ExprRegister != node->kind) break;
 
     /* Compare the middle destination register to the jump register. */
-    middle_reg = NaClGetExpRegister(node);
+    middle_reg = NaClGetExpRegisterInline(node);
     DEBUG(NaClLog(LOG_INFO, "middle reg = %s\n", NaClOpKindName(middle_reg)));
     if (middle_reg != jump_reg) break;
 
     if (InstLea == middle_inst->name) {
       /* Check that we have [%RBASE + %REG64-A] as second argument to lea */
       node = &nodes->node[op_2];
       if (ExprMemOffset != node->kind ||
           !NaClMemOffsetMatchesBaseIndex(nodes, op_2, state->base_register,
                                      and_64_reg)) {
         break;
@@ skipping 52 matching lines @@
                                           NaClJumpSets* jump_sets
                                           ) {
   uint8_t mask;
   NaClInstState* and_state;
   const NaClInst* and_inst;
   int op_1, op_2;
   NaClOpKind and_reg, jump_reg;
   NaClExpVector* nodes;
   NaClExp* node;
   assert(ExprRegister == reg->kind);
-  jump_reg = NaClGetExpRegister(reg);
+  jump_reg = NaClGetExpRegisterInline(reg);
 
   /* Do the following block exactly once. */
   do {
     /* Check that the jump is preceded with an AND. */
-    if (!NaClInstIterHasLookbackState(iter, 1)) break;
-    and_state = NaClInstIterGetLookbackState(iter, 1);
+    if (!NaClInstIterHasLookbackStateInline(iter, 1)) break;
+    and_state = NaClInstIterGetLookbackStateInline(iter, 1);
     and_inst = NaClInstStateInst(and_state);
     if (((and_state->num_opcode_bytes == 0) ||
          (0x83 != and_state->bytes.byte[and_state->num_prefix_bytes])) ||
         InstAnd != and_inst->name) break;
 
     /* Extract the values of the two operands for the and. */
     if (!NaClExtractBinaryOperandIndices(and_state, &op_1, &op_2)) break;
 
 
     /* Check that the register of the AND matches the jump. */
     nodes = NaClInstStateExpVector(and_state);
     node = &nodes->node[op_1];
     if (ExprRegister != node->kind) break;
-    and_reg = NaClGetExpRegister(node);
+    and_reg = NaClGetExpRegisterInline(node);
     if (jump_reg != and_reg) break;
 
     /* Check that the mask is ok. */
     mask = NaClGetJumpMask(state);
     assert(0 != mask);  /* alignment must be either 16 or 32. */
     node = &nodes->node[op_2];
     if (ExprConstant != node->kind || mask != node->value) break;
 
     /* If reached, indirect jump is properly masked. */
     DEBUG(NaClLog(LOG_INFO, "Protect register jump indirect\n"));
@@ skipping 107 matching lines @@
                            NaClPcAddress pc,
                            NaClInstState* inst_state,
                            NaClJumpSets* jump_sets) {
   if (pc < state->vbase || pc >= state->vlimit) {
     NaClValidatorInstMessage(LOG_ERROR, state, inst_state,
                              "Instruction pc out of range\n");
   } else {
     DEBUG(NaClLog(LOG_INFO,
                   "Add possible jump address: %"NACL_PRIxNaClPcAddress"\n",
                   pc));
-    NaClAddressSetAdd(jump_sets->possible_targets, pc, state);
+    NaClAddressSetAddInline(jump_sets->possible_targets, pc, state);
   }
 }
 
 void NaClJumpValidatorRememberIpOnly(NaClValidatorState* state,
                        NaClInstIter* iter,
                        NaClJumpSets* jump_sets) {
   NaClInstState* inst_state = state->cur_inst_state;
   NaClPcAddress pc = NaClInstStateVpc(inst_state);
   NaClRememberIp(state, pc, inst_state, jump_sets);
 }
@@ skipping 130 matching lines @@
      */
   } else {
     NaClJumpSets* jump_sets =
         (NaClJumpSets*)
         NaClGetValidatorLocalMemory((NaClValidator) NaClJumpValidator,
                                     state);
     DEBUG(NaClLog(LOG_INFO,
                   "Mark instruction as jump illegal: %"NACL_PRIxNaClPcAddress
                  "\n",
                  pc));
-    NaClAddressSetAdd(jump_sets->removed_targets, pc, state);
+    NaClAddressSetAddInline(jump_sets->removed_targets, pc, state);
   }
 }