Fix an integer overflow in opj_tcd_get_decoded_tile_size().

third_party/libopenjpeg20/tcd.c

Index: third_party/libopenjpeg20/tcd.c
diff --git a/third_party/libopenjpeg20/tcd.c b/third_party/libopenjpeg20/tcd.c
index 673633c09bc1601d6d4ea4c6fa59740e5b06d9b7..cd1c43921d4b9f77b559611d4bf4f6c73b57de20 100644
--- a/third_party/libopenjpeg20/tcd.c
+++ b/third_party/libopenjpeg20/tcd.c
@@ -1150,6 +1150,7 @@ OPJ_UINT32 opj_tcd_get_decoded_tile_size ( opj_tcd_t *p_tcd )
         opj_tcd_tilecomp_t * l_tile_comp = 00;
         opj_tcd_resolution_t * l_res = 00;
         OPJ_UINT32 l_size_comp, l_remaining;
+        OPJ_UINT32 l_temp;

[Lei Zhang, 2016/07/25 21:24:59]

Can we declare this inside the for-loop, since it's only used there?

[Oliver Chang, 2016/07/25 21:37:17]

The style of openjpeg seems to be to declare all variables at the beginning of
the function. Let's be consistent here?

 
         l_tile_comp = p_tcd->tcd_image->tiles->comps;
         l_img_comp = p_tcd->image->comps;
@@ -1167,7 +1168,18 @@ OPJ_UINT32 opj_tcd_get_decoded_tile_size ( opj_tcd_t *p_tcd )
                 }
 
                 l_res = l_tile_comp->resolutions + l_tile_comp->minimum_num_resolutions - 1;
-                l_data_size += l_size_comp * (OPJ_UINT32)((l_res->x1 - l_res->x0) * (l_res->y1 - l_res->y0));
+                l_temp = (OPJ_UINT32)((l_res->x1 - l_res->x0) * (l_res->y1 - l_res->y0)); /* x1*y1 can't overflow */
+
+                if (l_size_comp && ((OPJ_UINT32)-1) / l_size_comp < l_temp) {
+                        return (OPJ_UINT32)-1;
+                }
+                l_temp *= l_size_comp;
+
+                if (l_temp > ((OPJ_UINT32)-1) - l_data_size) {
+                        return (OPJ_UINT32)-1;
+                }
+                l_data_size += l_temp;
+
                 ++l_img_comp;
                 ++l_tile_comp;
         }
@@ -1362,7 +1374,7 @@ OPJ_BOOL opj_tcd_update_tile_data ( opj_tcd_t *p_tcd,
         OPJ_UINT32 l_stride, l_width,l_height;
 
         l_data_size = opj_tcd_get_decoded_tile_size(p_tcd);
-        if (l_data_size > p_dest_length) {
+        if (l_data_size == (OPJ_UINT32)-1 || l_data_size > p_dest_length) {
                 return OPJ_FALSE;
         }