Add additional sanity checks to RSA verification code.

firmware/lib/cryptolib/rsa_utility.c

 /* Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  *
  * Implementation of RSA utility functions.
  */
 
 #include "cryptolib.h"
 #include "stateful_util.h"
 #include "utility.h"
 
-int RSAProcessedKeySize(int algorithm) {
-  int key_len = siglen_map[algorithm];  /* Key length in
-                                         * bytes. */
-  /* Total size needed by a RSAPublicKey structure is =
-   *  2 * key_len bytes for the  n and rr arrays
-   *  + sizeof len + sizeof n0inv.
-   */
-  return (2 * key_len + sizeof(uint32_t) + sizeof(uint32_t));
+int RSAProcessedKeySize(unsigned int algorithm, int* out_size) {
+  int key_len; /* Key length in bytes. */
+  if (algorithm < kNumAlgorithms) {
+    key_len =  siglen_map[algorithm];
+    /* Total size needed by a RSAPublicKey structure is =
+     *  2 * key_len bytes for the  n and rr arrays
+     *  + sizeof len + sizeof n0inv.
+     */
+    *out_size = (2 * key_len + sizeof(uint32_t) + sizeof(uint32_t));
+    return 1;
+  }
+  return 0;
 }
 
 RSAPublicKey* RSAPublicKeyNew(void) {
   RSAPublicKey* key = (RSAPublicKey*) Malloc(sizeof(RSAPublicKey));
   key->n = NULL;
   key->rr = NULL;
   return key;
 }
 
 void RSAPublicKeyFree(RSAPublicKey* key) {
@@ skipping 37 matching lines @@
   }
 
   return key;
 }
 
 int RSAVerifyBinary_f(const uint8_t* key_blob,
                       const RSAPublicKey* key,
                       const uint8_t* buf,
                       uint64_t len,
                       const uint8_t* sig,
-                      int algorithm) {
+                      unsigned int algorithm) {
   RSAPublicKey* verification_key = NULL;
   uint8_t* digest = NULL;
   int key_size;
   int sig_size;
   int success;
 
   if (algorithm >= kNumAlgorithms)
     return 0;  /* Invalid algorithm. */
-  key_size = RSAProcessedKeySize(algorithm);
+  if (!RSAProcessedKeySize(algorithm, &key_size))
+    return 0;
   sig_size = siglen_map[algorithm];
 
   if (key_blob && !key)
     verification_key = RSAPublicKeyFromBuf(key_blob, key_size);
   else if (!key_blob && key)
     verification_key = (RSAPublicKey*) key;  /* Supress const warning. */
   else
     return 0; /* Both can't be NULL or non-NULL. */
 
+  /* Ensure we have a valid key. */
+  if (!verification_key)
+    return 0;
+
   digest = DigestBuf(buf, len, algorithm);
   success = RSAVerify(verification_key, sig, (uint32_t)sig_size,
                       (uint8_t)algorithm, digest);
 
   Free(digest);
   if (!key)
     RSAPublicKeyFree(verification_key);  /* Only free if we allocated it. */
   return success;
 }
 
 /* Version of RSAVerifyBinary_f() where instead of the raw binary blob
  * of data, its digest is passed as the argument. */
 int RSAVerifyBinaryWithDigest_f(const uint8_t* key_blob,
                                 const RSAPublicKey* key,
                                 const uint8_t* digest,
                                 const uint8_t* sig,
-                                int algorithm) {
+                                unsigned int algorithm) {
   RSAPublicKey* verification_key = NULL;
   int key_size;
   int sig_size;
   int success;
 
   if (algorithm >= kNumAlgorithms)
     return 0;  /* Invalid algorithm. */
-  key_size = RSAProcessedKeySize(algorithm);
+  if (!RSAProcessedKeySize(algorithm, &key_size))
+    return 0;
   sig_size = siglen_map[algorithm];
 
   if (key_blob && !key)
     verification_key = RSAPublicKeyFromBuf(key_blob, key_size);
   else if (!key_blob && key)
     verification_key = (RSAPublicKey*) key;  /* Supress const warning. */
   else
     return 0; /* Both can't be NULL or non-NULL. */
 
+  /* Ensure we have a valid key. */
+  if (!verification_key)
+    return 0;
+
   success = RSAVerify(verification_key, sig, (uint32_t)sig_size,
                       (uint8_t)algorithm, digest);
 
   if (!key)
     RSAPublicKeyFree(verification_key);  /* Only free if we allocated it. */
   return success;
 }