Use safe math when rendering line segments in AGG.

third_party/agg23/agg_rasterizer_scanline_aa.cpp

 
 //----------------------------------------------------------------------------
 // XYQ: 2006-01-22 Copied from AGG project.
 // This file uses only integer data, so it's suitable for all platforms.
 //----------------------------------------------------------------------------
 //----------------------------------------------------------------------------
 // Anti-Grain Geometry - Version 2.3
 // Copyright (C) 2002-2005 Maxim Shemanarev (http://www.antigrain.com)
 //
 // Permission to copy, use, modify, sell and distribute this software
@@ skipping 30 matching lines @@
 // ideas. Two functions - render_line and render_hline are the core of
 // the algorithm - they calculate the exact coverage of each pixel cell
 // of the polygon. I left these functions almost as is, because there's
 // no way to improve the perfection - hats off to David and his group!
 //
 // All other code is very different from the original.
 //
 //----------------------------------------------------------------------------
 #include <limits.h>
 #include "agg_rasterizer_scanline_aa.h"
+#include "third_party/base/numerics/safe_math.h"

[Tom Sepez, 2016/09/15 16:50:41]

Is this the first time we've introduced a dependency on our code into agg?

 namespace agg
 {
 AGG_INLINE void cell_aa::set_cover(int c, int a)
 {
     cover = c;
     area = a;
 }
 AGG_INLINE void cell_aa::add_cover(int c, int a)
 {
     cover += c;
@@ skipping 169 matching lines @@
         int cy = (y1 + y2) >> 1;
         render_line(x1, y1, cx, cy);
         render_line(cx, cy, x2, y2);
     }
     int dy = y2 - y1;
     int ey1 = y1 >> poly_base_shift;
     int ey2 = y2 >> poly_base_shift;
     int fy1 = y1 & poly_base_mask;
     int fy2 = y2 & poly_base_mask;
     int x_from, x_to;
-    int p, rem, mod, lift, delta, first, incr;
+    int rem, mod, lift, delta, first, incr;
     if(ey1 == ey2) {
         render_hline(ey1, x1, fy1, x2, fy2);
         return;
     }
     incr  = 1;
     if(dx == 0) {
         int ex = x1 >> poly_base_shift;
         int two_fx = (x1 - (ex << poly_base_shift)) << 1;
         int area;
         first = poly_base_size;
@@ skipping 10 matching lines @@
         area = two_fx * delta;
         while(ey1 != ey2) {
             m_cur_cell.set_cover(delta, area);
             ey1 += incr;
             set_cur_cell(ex, ey1);
         }
         delta = fy2 - poly_base_size + first;
         m_cur_cell.add_cover(delta, two_fx * delta);
         return;
     }
-    p     = (poly_base_size - fy1) * dx;
+    pdfium::base::CheckedNumeric<int> safeP = poly_base_size - fy1;
+    safeP *= dx;
+    if (!safeP.IsValid())
+      return;
     first = poly_base_size;
     if(dy < 0) {
-        p     = fy1 * dx;
-        first = 0;
-        incr  = -1;
-        dy    = -dy;
+      safeP = fy1;
+      safeP *= dx;
+      if (!safeP.IsValid())
+        return;
+      first = 0;
+      incr = -1;
+      dy = -dy;
     }
-    delta = p / dy;
-    mod   = p % dy;
+    delta = safeP.ValueOrDie() / dy;
+    mod = safeP.ValueOrDie() % dy;
     if(mod < 0) {
         delta--;
         mod += dy;
     }
     x_from = x1 + delta;
     render_hline(ey1, x1, fy1, x_from, first);
     ey1 += incr;
     set_cur_cell(x_from >> poly_base_shift, ey1);
     if(ey1 != ey2) {
-        p     = poly_base_size * dx;
-        lift  = p / dy;
-        rem   = p % dy;
-        if(rem < 0) {
-            lift--;
-            rem += dy;
+      safeP = static_cast<int>(poly_base_size);
+      safeP *= dx;
+      if (!safeP.IsValid())
+        return;
+      lift = safeP.ValueOrDie() / dy;
+      rem = safeP.ValueOrDie() % dy;
+      if (rem < 0) {
+        lift--;
+        rem += dy;
         }
         mod -= dy;
         while(ey1 != ey2) {
             delta = lift;
             mod  += rem;
             if (mod >= 0) {
                 mod -= dy;
                 delta++;
             }
             x_to = x_from + delta;
@@ skipping 173 matching lines @@
     }
     for(i = 0; i < m_sorted_y.size(); i++) {
         const sorted_y& cur_y = m_sorted_y[i];
         if(cur_y.num) {
             qsort_cells(m_sorted_cells.data() + cur_y.start, cur_y.num);
         }
     }
     m_sorted = true;
 }
 }