Make regress-crbug-514081 less flaky by having max serialization size

src/d8.cc

Index: src/d8.cc
diff --git a/src/d8.cc b/src/d8.cc
index fcdf2b6e43e0a57f57520d904099e75a63ee7af0..406607a4f816c8e1e3a7046ad247a54bf3dcd40f 100644
--- a/src/d8.cc
+++ b/src/d8.cc
@@ -2396,6 +2396,9 @@ bool Shell::SetOptions(int argc, char* argv[]) {
     } else if (strcmp(argv[i], "--enable-inspector") == 0) {
       options.enable_inspector = true;
       argv[i] = NULL;
+    } else if (strncmp(argv[i], "--max-serializer-memory-usage=", 30) == 0) {
+      options.max_serializer_memory_usage = atoi(argv[i] + 30);
+      argv[i] = NULL;
     }
   }
 
@@ -2490,8 +2493,11 @@ void Shell::EmptyMessageQueues(Isolate* isolate) {
 
 class Serializer : public ValueSerializer::Delegate {
  public:
-  explicit Serializer(Isolate* isolate)
-      : isolate_(isolate), serializer_(isolate, this) {}
+  Serializer(Isolate* isolate, size_t max_memory_usage)
+      : isolate_(isolate),
+        serializer_(isolate, this),
+        max_memory_usage_(max_memory_usage),
+        current_memory_usage_(0) {}
 
   Maybe<bool> WriteValue(Local<Context> context, Local<Value> value,
                          Local<Value> transfer) {
@@ -2542,6 +2548,10 @@ class Serializer : public ValueSerializer::Delegate {
 
   void* ReallocateBufferMemory(void* old_buffer, size_t size,
                                size_t* actual_size) override {
+    current_memory_usage_ += size;

[Michael Achenbach, 2017/02/15 09:35:24]

You could nest the addition within a condition about max_memory_usage_ != 0 as
well as it otherwise plays no roll.

I assume this might be ok for the purpose of testing and there's no place were
current_memory_usage_ is reduced?

[binji, 2017/02/15 20:31:55]

Right, I added a comment about that.

+    if (max_memory_usage_ != 0 && current_memory_usage_ > max_memory_usage_)
+      return nullptr;
+
     void* result = realloc(old_buffer, size);
     *actual_size = result ? size : 0;
     return result;
@@ -2619,6 +2629,8 @@ class Serializer : public ValueSerializer::Delegate {
   std::unique_ptr<SerializationData> data_;
   std::vector<Global<ArrayBuffer>> array_buffers_;
   std::vector<Global<SharedArrayBuffer>> shared_array_buffers_;
+  size_t max_memory_usage_; /* For testing when memory exhaustion. */

[Michael Achenbach, 2017/02/15 09:35:24]

nit: remove "when"?

[binji, 2017/02/15 20:31:55]

Done.

+  size_t current_memory_usage_;
 
   DISALLOW_COPY_AND_ASSIGN(Serializer);
 };
@@ -2667,7 +2679,7 @@ std::unique_ptr<SerializationData> Shell::SerializeValue(
     Isolate* isolate, Local<Value> value, Local<Value> transfer) {
   bool ok;
   Local<Context> context = isolate->GetCurrentContext();
-  Serializer serializer(isolate);
+  Serializer serializer(isolate, options.max_serializer_memory_usage);
   if (serializer.WriteValue(context, value, transfer).To(&ok)) {
     std::unique_ptr<SerializationData> data = serializer.Release();
     base::LockGuard<base::Mutex> lock_guard(workers_mutex_.Pointer());