ValueSerializer: Add more checks before trying to allocate memory for a dense array.

test/unittests/value-serializer-unittest.cc

 // Copyright 2016 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/value-serializer.h"
 
 #include <algorithm>
 #include <string>
 
 #include "include/v8.h"
@@ skipping 959 matching lines @@
         ASSERT_EQ(2, Array::Cast(*value)->Length());
         EXPECT_TRUE(
             EvaluateScriptForResultBool("typeof result[0] === 'undefined'"));
         EXPECT_TRUE(
             EvaluateScriptForResultBool("typeof result[1] === 'undefined'"));
         EXPECT_TRUE(EvaluateScriptForResultBool("!result.hasOwnProperty(0)"));
         EXPECT_TRUE(EvaluateScriptForResultBool("result.hasOwnProperty(1)"));
       });
 }
 
+TEST_F(ValueSerializerTest, DecodeInvalidOverLargeArray) {
+  // So large it couldn't exist in the V8 heap, and its size couldn't fit in a
+  // SMI on 32-bit systems (2^30).
+  InvalidDecodeTest({0xff, 0x09, 0x41, 0x80, 0x80, 0x80, 0x80, 0x04});
+  // Not so large, but there isn't enough data left in the buffer.
+  InvalidDecodeTest({0xff, 0x09, 0x41, 0x01});
+}
+
 TEST_F(ValueSerializerTest, RoundTripArrayWithNonEnumerableElement) {
   // Even though this array looks like [1,5,3], the 5 should be missing from the
   // perspective of structured clone, which only clones properties that were
   // enumerable.
   RoundTripTest(
       "(() => {"
       "  var x = [1,2,3];"
       "  Object.defineProperty(x, '1', {enumerable:false, value:5});"
       "  return x;"
       "})()",
@@ skipping 1363 matching lines @@
       "({ a: new ExampleHostObject(), get b() { return this.a; }})",
       [this](Local<Value> value) {
         EXPECT_TRUE(EvaluateScriptForResultBool(
             "result.a instanceof ExampleHostObject"));
         EXPECT_TRUE(EvaluateScriptForResultBool("result.a === result.b"));
       });
 }
 
 }  // namespace
 }  // namespace v8