openjpeg: Prevent an integer overflow in opj_jp2_apply_pclr.

third_party/libopenjpeg20/jp2.c

Index: third_party/libopenjpeg20/jp2.c
diff --git a/third_party/libopenjpeg20/jp2.c b/third_party/libopenjpeg20/jp2.c
index a6648f637ddbb30a18535587022400cdf666a64f..350803ae73e271ac0e6d104386ee6521df445a7f 100644
--- a/third_party/libopenjpeg20/jp2.c
+++ b/third_party/libopenjpeg20/jp2.c
@@ -990,7 +990,18 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
       assert( i == pcol ); // probably wrong?
       new_comps[i] = old_comps[cmp];
     }
-
+        /* Prevent integer overflow */
+        if (old_comps[cmp].h == 0 || old_comps[cmp].w > UINT_MAX / sizeof(OPJ_INT32) / old_comps[cmp].h) {
+            for (j = 0; j < i; ++j) {
+                if (new_comps[j].data) {
+                    opj_free(new_comps[j].data);
+                }
+            }
+            opj_free(new_comps);
+            new_comps = NULL;
+            return;
+        }
+        
 		/* Palette mapping: */
 		new_comps[i].data = (OPJ_INT32*)
 				opj_malloc(old_comps[cmp].w * old_comps[cmp].h * sizeof(OPJ_INT32));
@@ -1011,14 +1022,26 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
 		/* Palette mapping: */
 		cmp = cmap[i].cmp; pcol = cmap[i].pcol;
 		src = old_comps[cmp].data;
-    assert( src );
+        
+        /* Prevent null pointer access */
+        if (!old_comps[cmp].data || !new_comps[i].data) {
+            for (j = 0; j < nr_channels; ++j) {

[Oliver Chang, 2016/08/23 16:52:40]

these lines are very similar to 995-1000. maybe consolidate them into another
function?

+                if (new_comps[j].data) {
+                    opj_free(new_comps[j].data);
+                }
+            }
+            opj_free(new_comps);
+            new_comps = NULL;
+            return;
+        }
+        
 		max = new_comps[i].w * new_comps[i].h;
 
 		/* Direct use: */
     if(cmap[i].mtyp == 0) {
       assert( cmp == 0 ); // probably wrong.
       dst = new_comps[i].data;
-      assert( dst );
+      
       for(j = 0; j < max; ++j) {
         dst[j] = src[j];
       }
@@ -1026,7 +1049,7 @@ static void opj_jp2_apply_pclr(opj_image_t *image, opj_jp2_color_t *color)
     else {
       assert( i == pcol ); // probably wrong?
       dst = new_comps[i].data;
-      assert( dst );
+      
       for(j = 0; j < max; ++j) {
         /* The index */
         if((k = src[j]) < 0) k = 0; else if(k > top_k) k = top_k;