openjpeg: Prevent an integer overflow in opj_jp2_apply_pclr.

third_party/libopenjpeg20/jp2.c

 /*
  * The copyright in this software is being made available under the 2-clauses 
  * BSD License, included below. This software may be subject to other third 
  * party and contributor rights, including patent rights, and no such rights
  * are granted under this license.
  *
  * Copyright (c) 2002-2014, Universite catholique de Louvain (UCL), Belgium
  * Copyright (c) 2002-2014, Professor Benoit Macq
  * Copyright (c) 2001-2003, David Janssens
  * Copyright (c) 2002-2003, Yannick Verschueren
@@ skipping 972 matching lines @@
                 pcol = cmap[i].pcol; cmp = cmap[i].cmp;
 
                 /* Direct use */
     if(cmap[i].mtyp == 0){
       assert( pcol == 0 );
       new_comps[i] = old_comps[cmp];
     } else {
       assert( i == pcol ); // probably wrong?
       new_comps[i] = old_comps[cmp];
     }
-
+        /* Prevent integer overflow */
+        if (old_comps[cmp].h == 0 || old_comps[cmp].w > UINT_MAX / sizeof(OPJ_INT32) / old_comps[cmp].h) {
+            for (j = 0; j < i; ++j) {
+                if (new_comps[j].data) {
+                    opj_free(new_comps[j].data);
+                }
+            }
+            opj_free(new_comps);
+            new_comps = NULL;
+            return;
+        }
+        
                 /* Palette mapping: */
                 new_comps[i].data = (OPJ_INT32*)
                                 opj_malloc(old_comps[cmp].w * old_comps[cmp].h * sizeof(OPJ_INT32));
                 if (!new_comps[i].data) {
                         opj_free(new_comps);
                         new_comps = NULL;
                         /* FIXME no error code for opj_jp2_apply_pclr */
                         /* FIXME event manager error callback */
                         return;
                 }
                 new_comps[i].prec = channel_size[i];
                 new_comps[i].sgnd = channel_sign[i];
         }
 
         top_k = color->jp2_pclr->nr_entries - 1;
 
         for(i = 0; i < nr_channels; ++i) {
                 /* Palette mapping: */
                 cmp = cmap[i].cmp; pcol = cmap[i].pcol;
                 src = old_comps[cmp].data;
-    assert( src );
+        
+        /* Prevent null pointer access */
+        if (!old_comps[cmp].data || !new_comps[i].data) {
+            for (j = 0; j < nr_channels; ++j) {

[Oliver Chang, 2016/08/23 16:52:40]

these lines are very similar to 995-1000. maybe consolidate them into another
function?

+                if (new_comps[j].data) {
+                    opj_free(new_comps[j].data);
+                }
+            }
+            opj_free(new_comps);
+            new_comps = NULL;
+            return;
+        }
+        
                 max = new_comps[i].w * new_comps[i].h;
 
                 /* Direct use: */
     if(cmap[i].mtyp == 0) {
       assert( cmp == 0 ); // probably wrong.
       dst = new_comps[i].data;
-      assert( dst );
+      
       for(j = 0; j < max; ++j) {
         dst[j] = src[j];
       }
     }
     else {
       assert( i == pcol ); // probably wrong?
       dst = new_comps[i].data;
-      assert( dst );
+      
       for(j = 0; j < max; ++j) {
         /* The index */
         if((k = src[j]) < 0) k = 0; else if(k > top_k) k = top_k;
 
         /* The colour */
         dst[j] = (OPJ_INT32)entries[k * nr_channels + pcol];
         }
     }
         }
 
@@ skipping 2114 matching lines @@
   len = opj_stream_tell(cio)-lenp;
   opj_stream_skip(cio, lenp, p_manager);
   opj_write_bytes(l_data_header,len,4);/* L              */
   opj_stream_write_data(cio,l_data_header,4,p_manager);
   opj_stream_seek(cio, lenp+len,p_manager);
 
   return len;
 }
 #endif
 #endif /* USE_JPIP */