libtiff: Prevent a buffer overflow in function PixarLogDecode.

third_party/libtiff/tif_pixarlog.c

 /* $Id: tif_pixarlog.c,v 1.39 2012-12-10 17:27:13 tgl Exp $ */
 
 /*
  * Copyright (c) 1996-1997 Sam Leffler
  * Copyright (c) 1996 Pixar
  *
  * Permission to use, copy, modify, distribute, and sell this software and 
  * its documentation for any purpose is hereby granted without fee, provided
  * that (i) the above copyright notices and this permission notice appear in
  * all copies of the software and related documentation, and (ii) the names of
@@ skipping 439 matching lines @@
     }
 }
 
 /*
  * State block for each open TIFF
  * file using PixarLog compression/decompression.
  */
 typedef struct {
         TIFFPredictorState      predict;
         z_stream                stream;
+        tmsize_t                tbuf_size; /* only set/used on reading for now */
         uint16                  *tbuf; 
         uint16                  stride;
         int                     state;
         int                     user_datafmt;
         int                     quality;
 #define PLSTATE_INIT 1
 
         TIFFVSetMethod          vgetparent;     /* super-class method */
         TIFFVSetMethod          vsetparent;     /* super-class method */
 
@@ skipping 215 matching lines @@
             td->td_samplesperpixel : 1);
         tbuf_size = multiply_ms(multiply_ms(multiply_ms(sp->stride, td->td_imagewidth),
                                       td->td_rowsperstrip), sizeof(uint16));
         /* add one more stride in case input ends mid-stride */
         tbuf_size = add_ms(tbuf_size, sizeof(uint16) * sp->stride);
         if (tbuf_size == 0)
                 return (0);   /* TODO: this is an error return without error report through TIFFErrorExt */
         sp->tbuf = (uint16 *) _TIFFmalloc(tbuf_size);
         if (sp->tbuf == NULL)
                 return (0);
+        sp->tbuf_size = tbuf_size;
         if (sp->user_datafmt == PIXARLOGDATAFMT_UNKNOWN)
                 sp->user_datafmt = PixarLogGuessDataFmt(td);
         if (sp->user_datafmt == PIXARLOGDATAFMT_UNKNOWN) {
                 TIFFErrorExt(tif->tif_clientdata, module,
                         "PixarLog compression can't handle bits depth/data format combination (depth: %d)", 
                         td->td_bitspersample);
                 return (0);
         }
 
         if (inflateInit(&sp->stream) != Z_OK) {
@@ skipping 69 matching lines @@
         assert(sizeof(sp->stream.avail_out)==4);  /* if this assert gets raised,
             we need to simplify this code to reflect a ZLib that is likely updated
             to deal with 8byte memory sizes, though this code will respond
             apropriately even before we simplify it */
         sp->stream.avail_out = (uInt) (nsamples * sizeof(uint16));
         if (sp->stream.avail_out != nsamples * sizeof(uint16))
         {
                 TIFFErrorExt(tif->tif_clientdata, module, "ZLib cannot deal with buffers this size");
                 return (0);
         }
+        /* Check that we will not fill more than what was allocated */
+        if (sp->stream.avail_out > sp->tbuf_size)
+        {
+                TIFFErrorExt(tif->tif_clientdata, module, "sp->stream.avail_out > sp->tbuf_size");
+                return (0);
+        }
         do {
                 int state = inflate(&sp->stream, Z_PARTIAL_FLUSH);
                 if (state == Z_STREAM_END) {
                         break;                  /* XXX */
                 }
                 if (state == Z_DATA_ERROR) {
                         TIFFErrorExt(tif->tif_clientdata, module,
                             "Decoding error at scanline %lu, %s",
                             (unsigned long) tif->tif_row, sp->stream.msg);
                         if (inflateSync(&sp->stream) != Z_OK)
@@ skipping 639 matching lines @@
 #endif /* PIXARLOG_SUPPORT */
 
 /* vim: set ts=8 sts=8 sw=8 noet: */
 /*
  * Local Variables:
  * mode: c
  * c-basic-offset: 8
  * fill-column: 78
  * End:
  */