Fix an integer overflow in opj_tcd_get_decoded_tile_size().

third_party/libopenjpeg20/tcd.c

 /*
  * The copyright in this software is being made available under the 2-clauses 
  * BSD License, included below. This software may be subject to other third 
  * party and contributor rights, including patent rights, and no such rights
  * are granted under this license.
  *
  * Copyright (c) 2002-2014, Universite catholique de Louvain (UCL), Belgium
  * Copyright (c) 2002-2014, Professor Benoit Macq
  * Copyright (c) 2001-2003, David Janssens
  * Copyright (c) 2002-2003, Yannick Verschueren
@@ skipping 1132 matching lines @@
 }
 
 OPJ_UINT32 opj_tcd_get_decoded_tile_size ( opj_tcd_t *p_tcd )
 {
         OPJ_UINT32 i;
         OPJ_UINT32 l_data_size = 0;
         opj_image_comp_t * l_img_comp = 00;
         opj_tcd_tilecomp_t * l_tile_comp = 00;
         opj_tcd_resolution_t * l_res = 00;
         OPJ_UINT32 l_size_comp, l_remaining;
+        OPJ_UINT32 l_temp;

[Lei Zhang, 2016/07/25 21:24:59]

Can we declare this inside the for-loop, since it's only used there?

[Oliver Chang, 2016/07/25 21:37:17]

The style of openjpeg seems to be to declare all variables at the beginning of
the function. Let's be consistent here?

 
         l_tile_comp = p_tcd->tcd_image->tiles->comps;
         l_img_comp = p_tcd->image->comps;
 
         for (i=0;i<p_tcd->image->numcomps;++i) {
                 l_size_comp = l_img_comp->prec >> 3; /*(/ 8)*/
                 l_remaining = l_img_comp->prec & 7;  /* (%8) */
 
                 if(l_remaining) {
                         ++l_size_comp;
                 }
 
                 if (l_size_comp == 3) {
                         l_size_comp = 4;
                 }
 
                 l_res = l_tile_comp->resolutions + l_tile_comp->minimum_num_resolutions - 1;
-                l_data_size += l_size_comp * (OPJ_UINT32)((l_res->x1 - l_res->x0) * (l_res->y1 - l_res->y0));
+                l_temp = (OPJ_UINT32)((l_res->x1 - l_res->x0) * (l_res->y1 - l_res->y0)); /* x1*y1 can't overflow */
+
+                if (l_size_comp && ((OPJ_UINT32)-1) / l_size_comp < l_temp) {
+                        return (OPJ_UINT32)-1;
+                }
+                l_temp *= l_size_comp;
+
+                if (l_temp > ((OPJ_UINT32)-1) - l_data_size) {
+                        return (OPJ_UINT32)-1;
+                }
+                l_data_size += l_temp;
+
                 ++l_img_comp;
                 ++l_tile_comp;
         }
 
         return l_data_size;
 }
 
 OPJ_BOOL opj_tcd_encode_tile(   opj_tcd_t *p_tcd,
                                                         OPJ_UINT32 p_tile_no,
                                                         OPJ_BYTE *p_dest,
@@ skipping 174 matching lines @@
                                     )
 {
         OPJ_UINT32 i,j,k,l_data_size = 0;
         opj_image_comp_t * l_img_comp = 00;
         opj_tcd_tilecomp_t * l_tilec = 00;
         opj_tcd_resolution_t * l_res;
         OPJ_UINT32 l_size_comp, l_remaining;
         OPJ_UINT32 l_stride, l_width,l_height;
 
         l_data_size = opj_tcd_get_decoded_tile_size(p_tcd);
-        if (l_data_size > p_dest_length) {
+        if (l_data_size == (OPJ_UINT32)-1 || l_data_size > p_dest_length) {
                 return OPJ_FALSE;
         }
 
         l_tilec = p_tcd->tcd_image->tiles->comps;
         l_img_comp = p_tcd->image->comps;
 
         for (i=0;i<p_tcd->image->numcomps;++i) {
                 l_size_comp = l_img_comp->prec >> 3; /*(/ 8)*/
                 l_remaining = l_img_comp->prec & 7;  /* (%8) */
                 l_res = l_tilec->resolutions + l_img_comp->resno_decoded;
@@ skipping 827 matching lines @@
                                 }
                                 break;
                 }
 
                 ++l_img_comp;
                 ++l_tilec;
         }
 
         return OPJ_TRUE;
 }