Stop reporting OOM as errors in libpng fuzzers

testing/libfuzzer/fuzzers/libpng_read_fuzzer.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <vector>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #define PNG_INTERNAL
 #include "third_party/libpng/png.h"
 
+void* limited_malloc(png_structp, png_alloc_size_t size) {
+  // libpng may allocate large amounts of memory that the fuzzer reports as
+  // an error. In order to silence these errors, make libpng fail when trying
+  // to allocate a large amount.
+  // This number is chosen to match the default png_user_chunk_malloc_max.
+  if (size > 8000000)
+    return nullptr;
+
+  return malloc(size);
+}
+
+void default_free(png_structp, png_voidp ptr) {
+  return free(ptr);
+}
+
 #ifndef PNG_FUZZ_PROGRESSIVE
 
 // Read sequentially, with png_read_row.
 struct BufState {
   const uint8_t* data;
   size_t bytes_left;
 };
 
 void user_read_data(png_structp png_ptr, png_bytep data, png_size_t length) {
   BufState* buf_state = static_cast<BufState*>(png_get_io_ptr(png_ptr));
@@ skipping 26 matching lines @@
   png_structp png_ptr = png_create_read_struct
     (PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr);
   assert(png_ptr);
 
 #ifdef MEMORY_SANITIZER
   // To avoid OOM with MSan (crbug.com/648073). These values are recommended as
   // safe settings by https://github.com/glennrp/libpng/blob/libpng16/pngusr.dfa
   png_set_user_limits(png_ptr, 65535, 65535);
 #endif
 
+  // Not all potential OOM are due to images with large widths and heights.
+  // Use a custom allocator that fails for large allocations.
+  png_set_mem_fn(png_ptr, nullptr, limited_malloc, default_free);
+
   png_set_crc_action(png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE);
 
   png_infop info_ptr = png_create_info_struct(png_ptr);
   assert(info_ptr);
 
   base::ScopedClosureRunner struct_deleter(base::Bind(
         &png_destroy_read_struct, &png_ptr, &info_ptr, nullptr));
 
 #ifdef PNG_FUZZ_PROGRESSIVE
   if (setjmp(png_jmpbuf(png_ptr))) {
@@ skipping 45 matching lines @@
 
   for (int pass = 0; pass < passes; ++pass) {
     for (png_uint_32 y = 0; y < height; ++y) {
       png_read_row(png_ptr, static_cast<png_bytep>(row), NULL);
     }
   }
 #endif  // PNG_FUZZ_PROGRESSIVE
 
   return 0;
 }