Fix a couple of divide by zero crashes in PNG/TIFF predictors.

core/src/fxcodec/codec/fx_codec_flate.cpp

 // Copyright 2014 PDFium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
  
 // Original code copyright 2014 Foxit Software Inc. http://www.foxitsoftware.com
 
+#include "../../../../third_party/base/nonstd_unique_ptr.h"
+#include "../../../include/fxcodec/fx_codec.h"
 #include "../../fx_zlib.h"
-#include "../../../include/fxcodec/fx_codec.h"
 #include "codec_int.h"
+
 extern "C"
 {
     static void* my_alloc_func (void* opaque, unsigned int items, unsigned int size)
     {
         return FX_Alloc(FX_BYTE, items * size);
     }
     static void   my_free_func  (void* opaque, void* address)
     {
         FX_Free(address);
     }
@@ skipping 202 matching lines @@
     int pb = FXSYS_abs(p - b);
     int pc = FXSYS_abs(p - c);
     if (pa <= pb && pa <= pc) {
         return (FX_BYTE)a;
     }
     if (pb <= pc) {
         return (FX_BYTE)b;
     }
     return (FX_BYTE)c;
 }
-static void PNG_PredictorEncode(FX_LPBYTE& data_buf, FX_DWORD& data_size, int predictor, int Colors, int BitsPerComponent, int Columns)
+static FX_BOOL PNG_PredictorEncode(FX_LPBYTE& data_buf, FX_DWORD& data_size,
+                                   int predictor, int Colors,
+                                   int BitsPerComponent, int Columns)
 {
-    int BytesPerPixel = (Colors * BitsPerComponent + 7) / 8;
-    int row_size = (Colors * BitsPerComponent * Columns + 7) / 8;
-    int row_count = (data_size + row_size - 1) / row_size;
-    int last_row_size = data_size % row_size;
+    const int BytesPerPixel = (Colors * BitsPerComponent + 7) / 8;
+    const int row_size = (Colors * BitsPerComponent * Columns + 7) / 8;
+    if (row_size <= 0)
+        return FALSE;
+    const int row_count = (data_size + row_size - 1) / row_size;
+    const int last_row_size = data_size % row_size;
     FX_LPBYTE dest_buf = FX_Alloc( FX_BYTE, (row_size + 1) * row_count);
-    if (dest_buf == NULL) {
-        return;
-    }
+    if (dest_buf == NULL)
+        return FALSE;
     int byte_cnt = 0;
     FX_LPBYTE pSrcData = data_buf;
     FX_LPBYTE pDestData = dest_buf;
     for (int row = 0; row < row_count; row++) {
         if (predictor == 10) {
             pDestData[0] = 0;
             int move_size = row_size;
             if (move_size * (row + 1) > (int)data_size) {
                 move_size = data_size - (move_size * row);
             }
@@ skipping 59 matching lines @@
                     break;
             }
             byte_cnt++;
         }
         pDestData += (row_size + 1);
         pSrcData += row_size;
     }
     FX_Free(data_buf);
     data_buf = dest_buf;
     data_size = (row_size + 1) * row_count - (last_row_size > 0 ? (row_size - last_row_size) : 0);
+    return TRUE;
 }
 static void PNG_PredictLine(FX_LPBYTE pDestData, FX_LPCBYTE pSrcData, FX_LPCBYTE pLastLine,
                             int bpc, int nColors, int nPixels)
 {
     int row_size = (nPixels * bpc * nColors + 7) / 8;
     int BytesPerPixel = (bpc * nColors + 7) / 8;
     FX_BYTE tag = pSrcData[0];
     if (tag == 0) {
         FXSYS_memmove32(pDestData, pSrcData + 1, row_size);
         return;
@@ skipping 44 matching lines @@
                     }
                     pDestData[byte] = raw_byte + PaethPredictor(left, up, upper_left);
                     break;
                 }
             default:
                 pDestData[byte] = raw_byte;
                 break;
         }
     }
 }
-static void PNG_Predictor(FX_LPBYTE& data_buf, FX_DWORD& data_size,
-                          int Colors, int BitsPerComponent, int Columns)
+static FX_BOOL PNG_Predictor(FX_LPBYTE& data_buf, FX_DWORD& data_size,
+                             int Colors, int BitsPerComponent, int Columns)
 {
-    int BytesPerPixel = (Colors * BitsPerComponent + 7) / 8;
-    int row_size = (Colors * BitsPerComponent * Columns + 7) / 8;
-    int row_count = (data_size + row_size) / (row_size + 1);
-    int last_row_size = data_size % (row_size + 1);
+    const int BytesPerPixel = (Colors * BitsPerComponent + 7) / 8;
+    const int row_size = (Colors * BitsPerComponent * Columns + 7) / 8;
+    if (row_size <= 0)
+        return FALSE;
+    const int row_count = (data_size + row_size) / (row_size + 1);
+    const int last_row_size = data_size % (row_size + 1);
     FX_LPBYTE dest_buf = FX_Alloc( FX_BYTE, row_size * row_count);
-    if (dest_buf == NULL) {
-        return;
-    }
+    if (dest_buf == NULL)
+        return FALSE;
     int byte_cnt = 0;
     FX_LPBYTE pSrcData = data_buf;
     FX_LPBYTE pDestData = dest_buf;
     for (int row = 0; row < row_count; row ++) {
         FX_BYTE tag = pSrcData[0];
         byte_cnt++;
         if (tag == 0) {
             int move_size = row_size;
             if ((row + 1) * (move_size + 1) > (int)data_size) {
                 move_size = last_row_size - 1;
@@ skipping 56 matching lines @@
                     break;
             }
             byte_cnt++;
         }
         pSrcData += row_size + 1;
         pDestData += row_size;
     }
     FX_Free(data_buf);
     data_buf = dest_buf;
     data_size = row_size * row_count - (last_row_size > 0 ? (row_size + 1 - last_row_size) : 0);
+    return TRUE;
 }
 static void TIFF_PredictorEncodeLine(FX_LPBYTE dest_buf, int row_size, int BitsPerComponent, int Colors, int Columns)
 {
     int BytesPerPixel = BitsPerComponent * Colors / 8;
     if (BitsPerComponent < 8) {
         FX_BYTE mask = 0x01;
         if (BitsPerComponent == 2) {
             mask = 0x03;
         } else if (BitsPerComponent == 4) {
             mask = 0x0F;
@@ skipping 18 matching lines @@
         }
     } else {
         for (int i = row_size - BytesPerPixel; i >= BytesPerPixel; i -= BytesPerPixel) {
             FX_WORD pixel = (dest_buf[i] << 8) | dest_buf[i + 1];
             pixel -= (dest_buf[i - BytesPerPixel] << 8) | dest_buf[i - BytesPerPixel + 1];
             dest_buf[i] = pixel >> 8;
             dest_buf[i + 1] = (FX_BYTE)pixel;
         }
     }
 }
-static void TIFF_PredictorEncode(FX_LPBYTE& data_buf, FX_DWORD& data_size,
-                                 int Colors, int BitsPerComponent, int Columns)
+static FX_BOOL TIFF_PredictorEncode(FX_LPBYTE& data_buf, FX_DWORD& data_size,
+                                    int Colors, int BitsPerComponent, int Columns)
 {
     int row_size = (Colors * BitsPerComponent * Columns + 7) / 8;
-    int row_count = (data_size + row_size - 1) / row_size;
-    int last_row_size = data_size % row_size;
+    if (row_size == 0)
+        return FALSE;
+    const int row_count = (data_size + row_size - 1) / row_size;
+    const int last_row_size = data_size % row_size;
     for (int row = 0; row < row_count; row++) {
         FX_LPBYTE scan_line = data_buf + row * row_size;
         if ((row + 1) * row_size > (int)data_size) {
             row_size = last_row_size;
         }
         TIFF_PredictorEncodeLine(scan_line, row_size, BitsPerComponent, Colors, Columns);
     }
+    return TRUE;
 }
 static void TIFF_PredictLine(FX_LPBYTE dest_buf, int row_size, int BitsPerComponent, int Colors, int Columns)
 {
     if (BitsPerComponent == 1) {
         int row_bits = FX_MIN(BitsPerComponent * Colors * Columns, row_size * 8);
         int index_pre = 0;
         int col_pre = 0;
         for(int i = 1; i < row_bits; i ++) {
             int col = i % 8;
             int index = i / 8;
@@ skipping 14 matching lines @@
             pixel += (dest_buf[i] << 8) | dest_buf[i + 1];
             dest_buf[i] = pixel >> 8;
             dest_buf[i + 1] = (FX_BYTE)pixel;
         }
     } else {
         for (int i = BytesPerPixel; i < row_size; i ++) {
             dest_buf[i] += dest_buf[i - BytesPerPixel];
         }
     }
 }
-static void TIFF_Predictor(FX_LPBYTE& data_buf, FX_DWORD& data_size,
-                           int Colors, int BitsPerComponent, int Columns)
+static FX_BOOL TIFF_Predictor(FX_LPBYTE& data_buf, FX_DWORD& data_size,
+                              int Colors, int BitsPerComponent, int Columns)
 {
     int row_size = (Colors * BitsPerComponent * Columns + 7) / 8;
-    int row_count = (data_size + row_size - 1) / row_size;
-    int last_row_size = data_size % row_size;
+    if (row_size == 0)
+        return FALSE;
+    const int row_count = (data_size + row_size - 1) / row_size;
+    const int last_row_size = data_size % row_size;
     for (int row = 0; row < row_count; row ++) {
         FX_LPBYTE scan_line = data_buf + row * row_size;
         if ((row + 1) * row_size > (int)data_size) {
             row_size = last_row_size;
         }
         TIFF_PredictLine(scan_line, row_size, BitsPerComponent, Colors, Columns);
     }
+    return TRUE;
 }
 class CCodec_FlateScanlineDecoder : public CCodec_ScanlineDecoder
 {
 public:
     CCodec_FlateScanlineDecoder();
     ~CCodec_FlateScanlineDecoder();
     FX_BOOL             Create(FX_LPCBYTE src_buf, FX_DWORD src_size, int width, int height, int nComps, int bpc,
                        int predictor, int Colors, int BitsPerComponent, int Columns);
     virtual void                Destroy()
     {
@@ skipping 272 matching lines @@
         int nComps, int bpc, int predictor, int Colors, int BitsPerComponent, int Columns)
 {
     CCodec_FlateScanlineDecoder* pDecoder = new CCodec_FlateScanlineDecoder;
     pDecoder->Create(src_buf, src_size, width, height, nComps, bpc, predictor, Colors, BitsPerComponent, Columns);
     return pDecoder;
 }
 FX_DWORD CCodec_FlateModule::FlateOrLZWDecode(FX_BOOL bLZW, const FX_BYTE* src_buf, FX_DWORD src_size, FX_BOOL bEarlyChange,
         int predictor, int Colors, int BitsPerComponent, int Columns,
         FX_DWORD estimated_size, FX_LPBYTE& dest_buf, FX_DWORD& dest_size)
 {
-    CLZWDecoder* pDecoder = NULL;
     dest_buf = NULL;
     FX_DWORD offset = 0;
     int predictor_type = 0;
     if (predictor) {
         if (predictor >= 10) {
             predictor_type = 2;
         } else if (predictor == 2) {
             predictor_type = 1;
         }
     }
     if (bLZW) {
-        pDecoder = new CLZWDecoder;
-        dest_size = (FX_DWORD) - 1;
-        offset = src_size;
-        int err = pDecoder->Decode(NULL, dest_size, src_buf, offset, bEarlyChange);
-        delete pDecoder;
-        if (err || dest_size == 0 || dest_size + 1 < dest_size) {
-            return (FX_DWORD) - 1;
+        {
+            nonstd::unique_ptr<CLZWDecoder> decoder(new CLZWDecoder);
+            dest_size = (FX_DWORD) - 1;
+            offset = src_size;
+            int err = decoder->Decode(NULL, dest_size, src_buf, offset,
+                                      bEarlyChange);
+            if (err || dest_size == 0 || dest_size + 1 < dest_size) {
+                return -1;
+            }
         }
-        pDecoder = new CLZWDecoder;
-        dest_buf = FX_Alloc( FX_BYTE, dest_size + 1);
-        if (dest_buf == NULL) {
-            return -1;
+        {
+            nonstd::unique_ptr<CLZWDecoder> decoder(new CLZWDecoder);
+            dest_buf = FX_Alloc( FX_BYTE, dest_size + 1);
+            if (dest_buf == NULL) {
+                return -1;
+            }
+            dest_buf[dest_size] = '\0';
+            decoder->Decode(dest_buf, dest_size, src_buf, offset, bEarlyChange);
         }
-        dest_buf[dest_size] = '\0';
-        pDecoder->Decode(dest_buf, dest_size, src_buf, offset, bEarlyChange);
-        delete pDecoder;
     } else {
         FlateUncompress(src_buf, src_size, estimated_size, dest_buf, dest_size, offset);
     }
     if (predictor_type == 0) {
         return offset;
     }
+    FX_BOOL ret = TRUE;
     if (predictor_type == 2) {
-        PNG_Predictor(dest_buf, dest_size, Colors, BitsPerComponent, Columns);
+        ret = PNG_Predictor(dest_buf, dest_size, Colors, BitsPerComponent,
+                            Columns);
     } else if (predictor_type == 1) {
-        TIFF_Predictor(dest_buf, dest_size, Colors, BitsPerComponent, Columns);
+        ret = TIFF_Predictor(dest_buf, dest_size, Colors, BitsPerComponent,
+                             Columns);
     }
-    return offset;
+    return ret ? offset : -1;
 }
 FX_BOOL CCodec_FlateModule::Encode(const FX_BYTE* src_buf, FX_DWORD src_size,
                                    int predictor, int Colors, int BitsPerComponent, int Columns,
                                    FX_LPBYTE& dest_buf, FX_DWORD& dest_size)
 {
     if (predictor != 2 && predictor < 10) {
         return Encode(src_buf, src_size, dest_buf, dest_size);
     }
-    FX_BOOL ret = FALSE;
     FX_LPBYTE pSrcBuf = NULL;
     pSrcBuf = FX_Alloc(FX_BYTE, src_size);
     if (pSrcBuf == NULL) {
         return FALSE;
     }
     FXSYS_memcpy32(pSrcBuf, src_buf, src_size);
+    FX_BOOL ret = TRUE;
     if (predictor == 2) {
-        TIFF_PredictorEncode(pSrcBuf, src_size, Colors, BitsPerComponent, Columns);
+        ret = TIFF_PredictorEncode(pSrcBuf, src_size, Colors, BitsPerComponent,
+                                   Columns);
     } else if (predictor >= 10) {
-        PNG_PredictorEncode(pSrcBuf, src_size, predictor, Colors, BitsPerComponent, Columns);
+        ret = PNG_PredictorEncode(pSrcBuf, src_size, predictor, Colors,
+                                  BitsPerComponent, Columns);
     }
-    ret = Encode(pSrcBuf, src_size, dest_buf, dest_size);
+    if (ret)
+        ret = Encode(pSrcBuf, src_size, dest_buf, dest_size);
     FX_Free(pSrcBuf);
     return ret;
 }
 FX_BOOL CCodec_FlateModule::Encode(FX_LPCBYTE src_buf, FX_DWORD src_size, FX_LPBYTE& dest_buf, FX_DWORD& dest_size)
 {
     dest_size = src_size + src_size / 1000 + 12;
     dest_buf = FX_Alloc( FX_BYTE, dest_size);
     if (dest_buf == NULL) {
         return FALSE;
     }
     unsigned long temp_size = dest_size;
     FPDFAPI_FlateCompress(dest_buf, &temp_size, src_buf, src_size);
     dest_size = (FX_DWORD)temp_size;
     return TRUE;
 }