[qcms] Fix integer overflow in lut_inverse_interp16

third_party/qcms/src/transform_util.c

 //  qcms
 //  Copyright (C) 2009 Mozilla Foundation
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the "Software"),
 // to deal in the Software without restriction, including without limitation
 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
 // and/or sell copies of the Software, and to permit persons to whom the Software
 // is furnished to do so, subject to the following conditions:
 //
@@ skipping 322 matching lines @@
 
         // There are no zeros at the beginning and we are trying to find a zero, so
         // return anything. It seems zero would be the less destructive choice
         /* I'm not sure that this makes sense, but oh well... */
         if (NumZeroes == 0 && Value == 0)
             return 0;
 
         // Does the curve belong to this case?
         if (NumZeroes > 1 || NumPoles > 1)
         {
-                int a, b, sample;
+                float a, b;
+                int sample;
 
                 // Identify if value fall downto 0 or FFFF zone
                 if (Value == 0) return 0;
                 // if (Value == 0xFFFF) return 0xFFFF;
                 sample = (length-1) * ((double) Value * (1./65535.));
                 if (LutTable[sample] == 0xffff)
                     return 0xffff;
 
                 // else restrict to valid zone
 
-                a = ((NumZeroes-1) * 0xFFFF) / (length-1);
-                b = ((length-1 - NumPoles) * 0xFFFF) / (length-1);
+                a = ((NumZeroes-1) * 65535.f) / (length-1);
+                b = ((length-1 - NumPoles) * 65535.f) / (length-1);
 
-                l = a - 1;
-                r = b + 1;
+                l = ((int)a) - 1;
+                r = ((int)b) + 1;
 
                 // Ensure a valid binary search range
 
                 if (l < 1)
                     l = 1;
                 if (r > 0x10000)
                     r = 0x10000;
 
                 // If the search range is inverted due to degeneracy,
                 // deem LutTable non-invertible in this search range.
@@ skipping 301 matching lines @@
     if (!curve || curve->type != PARAMETRIC_CURVE_TYPE)
         return 0;
 
     size = COUNT_TO_LENGTH[curve->count];
 
     if (data)
         memcpy(data, curve->parameter, size * sizeof(float));
 
     return size;
 }