Fix UBSan errors (signed integer overflow)

webrtc/modules/audio_coding/codecs/ilbc/smooth.c

Index: webrtc/modules/audio_coding/codecs/ilbc/smooth.c
diff --git a/webrtc/modules/audio_coding/codecs/ilbc/smooth.c b/webrtc/modules/audio_coding/codecs/ilbc/smooth.c
index 58b37ee31cc99a4959c8f1f1b5d4ea93521bdd96..7901fb823e9834c5c3dc29b3c15898d536b5adea 100644
--- a/webrtc/modules/audio_coding/codecs/ilbc/smooth.c
+++ b/webrtc/modules/audio_coding/codecs/ilbc/smooth.c
@@ -31,7 +31,7 @@ void WebRtcIlbcfix_Smooth(
     int16_t *surround  /* (i) The approximation from the
                                 surrounding sequences */
                           ) {
-  int16_t maxtot, scale, scale1, scale2;
+  int16_t scale, scale1, scale2;
   int16_t A, B, C, denomW16;
   int32_t B_W32, denom, num;
   int32_t errs;
@@ -40,18 +40,20 @@ void WebRtcIlbcfix_Smooth(
   int16_t w11prim;
   int16_t bitsw00, bitsw10, bitsw11;
   int32_t w11w00, w10w10, w00w00;
-  int16_t max1, max2;
+  uint32_t max1, max2;
 
   /* compute some inner products (ensure no overflow by first calculating proper scale factor) */
 
   w00 = w10 = w11 = 0;
 
-  max1=WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL);
-  max2=WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL);
-  maxtot=WEBRTC_SPL_MAX(max1, max2);
-
-  scale=WebRtcSpl_GetSizeInBits(maxtot);
-  scale = (int16_t)(2 * scale) - 26;
+  // Calculate a right shift that will let us sum ENH_BLOCKL pairwise products
+  // of values from the two sequences without overflowing an int32_t. (The +1
+  // in max1 and max2 are because WebRtcSpl_MaxAbsValueW16 will return 2**15 -
+  // 1 if the input array contains -2**15.)
+  max1 = WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL) + 1;
+  max2 = WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL) + 1;
+  scale = (64 - 31) -
+          WebRtcSpl_CountLeadingZeros64((max1 * max2) * (uint64_t)ENH_BLOCKL);

[tlegrand-webrtc, 2016/05/26 12:18:21]

I think you need to keep the "max 26 bits" not to get overflow later on.

[kwiberg-webrtc, 2016/05/26 13:52:10]

No, this piece of code works mostly the same as the other one. Except for
actually checking the max of both arrays instead of just one. And the number of
products to sum (ENH_BLOCKL) is a constant, not a variable, but still assumed to
be at most 32.

Curiously, ENH_BLOCKL is in fact 80. Meaning that the old code systematically
underestimated the required right-shift by more than one step.

   scale=WEBRTC_SPL_MAX(0, scale);
 
   w00=WebRtcSpl_DotProductWithScale(current,current,ENH_BLOCKL,scale);