Add additional sanity checks to RSA verification code.

host/lib/file_keys.c

 /* Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  *
  * Utility functions for file and key handling.
  */
 
 #include "file_keys.h"
 
 #include <fcntl.h>
@@ skipping 64 matching lines @@
           SHA1_BLOCK_SIZE)
     DigestUpdate(&ctx, data, len);
   if (len != -1)
     DigestUpdate(&ctx, data, len);
   digest = DigestFinal(&ctx);
   close(input_fd);
   return digest;
 }
 
 uint8_t* SignatureFile(const char* input_file, const char* key_file,
-                       int algorithm) {
+                       unsigned int algorithm) {
   char* sign_utility = "./sign_data.sh";
   char* cmd;  /* Command line to invoke. */
   int cmd_len;
   FILE* cmd_out;  /* File descriptor to command output. */
   uint8_t* signature = NULL;
   int signature_size = siglen_map[algorithm];
 
   /* Build command line:
    * sign_data.sh <algorithm> <key file> <input file>
    */
   cmd_len = (strlen(sign_utility) + 1 + /* +1 for space. */
              2 + 1 + /* For [algorithm]. */
              strlen(key_file) + 1 + /* +1 for space. */
              strlen(input_file) +
              1);  /* For the trailing '\0'. */
   cmd = (char*) Malloc(cmd_len);
-  snprintf(cmd, cmd_len, "%s %d %s %s", sign_utility, algorithm, key_file,
+  snprintf(cmd, cmd_len, "%s %u %s %s", sign_utility, algorithm, key_file,
            input_file);
   cmd_out = popen(cmd, "r");
   Free(cmd);
   if (!cmd_out) {
     VBDEBUG(("Couldn't execute: %s\n", cmd));
     return NULL;
   }
 
   signature = (uint8_t*) Malloc(signature_size);
   if (fread(signature, signature_size, 1, cmd_out) != 1) {
     VBDEBUG(("Couldn't read signature.\n"));
     pclose(cmd_out);
     Free(signature);
     return NULL;
   }
 
   pclose(cmd_out);
   return signature;
 }