Fix UBSan errors (signed integer overflow)

webrtc/modules/audio_coding/codecs/ilbc/smooth.c

 /*
  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
  * find the smoothed output data
  *---------------------------------------------------------------*/
 
 void WebRtcIlbcfix_Smooth(
     int16_t *odata,   /* (o) smoothed output */
     int16_t *current,  /* (i) the un enhanced residual for
                                 this block */
     int16_t *surround  /* (i) The approximation from the
                                 surrounding sequences */
                           ) {
-  int16_t maxtot, scale, scale1, scale2;
+  int16_t scale, scale1, scale2;
   int16_t A, B, C, denomW16;
   int32_t B_W32, denom, num;
   int32_t errs;
   int32_t w00,w10,w11, endiff, crit;
   int32_t w00prim, w10prim, w11_div_w00;
   int16_t w11prim;
   int16_t bitsw00, bitsw10, bitsw11;
   int32_t w11w00, w10w10, w00w00;
-  int16_t max1, max2;
+  uint32_t max1, max2;
 
   /* compute some inner products (ensure no overflow by first calculating proper scale factor) */
 
   w00 = w10 = w11 = 0;
 
-  max1=WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL);
-  max2=WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL);
-  maxtot=WEBRTC_SPL_MAX(max1, max2);
-
-  scale=WebRtcSpl_GetSizeInBits(maxtot);
-  scale = (int16_t)(2 * scale) - 26;
+  // Calculate a right shift that will let us sum ENH_BLOCKL pairwise products
+  // of values from the two sequences without overflowing an int32_t. (The +1
+  // in max1 and max2 are because WebRtcSpl_MaxAbsValueW16 will return 2**15 -
+  // 1 if the input array contains -2**15.)
+  max1 = WebRtcSpl_MaxAbsValueW16(current, ENH_BLOCKL) + 1;
+  max2 = WebRtcSpl_MaxAbsValueW16(surround, ENH_BLOCKL) + 1;
+  scale = (64 - 31) -
+          WebRtcSpl_CountLeadingZeros64((max1 * max2) * (uint64_t)ENH_BLOCKL);

[tlegrand-webrtc, 2016/05/26 12:18:21]

I think you need to keep the "max 26 bits" not to get overflow later on.

[kwiberg-webrtc, 2016/05/26 13:52:10]

No, this piece of code works mostly the same as the other one. Except for
actually checking the max of both arrays instead of just one. And the number of
products to sum (ENH_BLOCKL) is a constant, not a variable, but still assumed to
be at most 32.

Curiously, ENH_BLOCKL is in fact 80. Meaning that the old code systematically
underestimated the required right-shift by more than one step.

   scale=WEBRTC_SPL_MAX(0, scale);
 
   w00=WebRtcSpl_DotProductWithScale(current,current,ENH_BLOCKL,scale);
   w11=WebRtcSpl_DotProductWithScale(surround,surround,ENH_BLOCKL,scale);
   w10=WebRtcSpl_DotProductWithScale(surround,current,ENH_BLOCKL,scale);
 
   if (w00<0) w00 = WEBRTC_SPL_WORD32_MAX;
   if (w11<0) w11 = WEBRTC_SPL_WORD32_MAX;
 
   /* Rescale w00 and w11 to w00prim and w11prim, so that w00prim/w11prim
@@ skipping 133 matching lines @@
     }
 
     /* create smoothed sequence */
 
     WebRtcSpl_ScaleAndAddVectors(surround, A, 9,
                                 current, B, 14,
                                 odata, ENH_BLOCKL);
   }
   return;
 }