Make finalization of recursive function types more robust, especially since

runtime/vm/class_finalizer.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/class_finalizer.h"
 
 #include "vm/code_generator.h"
 #include "vm/flags.h"
 #include "vm/hash_table.h"
 #include "vm/heap.h"
@@ skipping 797 matching lines @@
         // The parameterized_type is raw. Set its argument vector to null, which
         // is more efficient in type tests.
         full_arguments = TypeArguments::null();
       }
       type.set_arguments(full_arguments);
     } else {
       ASSERT(full_arguments.IsNull());  // Use null vector for raw type.
     }
   }
 
-  // Self referencing types may get finalized indirectly.
-  if (!type.IsFinalized()) {
-    ASSERT(full_arguments.IsNull() ||
-           !full_arguments.IsRaw(0, num_type_arguments));
-    if (FLAG_trace_type_finalization) {
-      THR_Print("Marking type '%s' as finalized for class '%s'\n",
-                String::Handle(zone, type.Name()).ToCString(),
-                String::Handle(zone, cls.Name()).ToCString());
-    }
-    // Mark the type as finalized.
-    type.SetIsFinalized();
-    // Do not yet remove the type from the pending_types array.
-  }
+  ASSERT(full_arguments.IsNull() ||
+         !full_arguments.IsRaw(0, num_type_arguments));
   return full_arguments.IsNull() ? 0 : full_arguments.Length();
 }
 
 
 // Finalize the type argument vector 'arguments' of the type defined by the
 // class 'cls' parameterized with the type arguments 'cls_args'.
 // The vector 'cls_args' is already initialized as a subvector at the correct
 // position in the passed in 'arguments' vector.
 // The subvector 'cls_args' has length cls.NumTypeParameters() and starts at
 // offset cls.NumTypeArguments() - cls.NumTypeParameters() of the 'arguments'
@@ skipping 377 matching lines @@
   // allocated yet, and if canonicalization is required.
   const bool is_root_type =
       (pending_types == NULL) && (finalization >= kCanonicalize);
   if (is_root_type) {
     pending_types = new PendingTypes(zone, 4);
   }
 
   const intptr_t num_expanded_type_arguments =
       ExpandAndFinalizeTypeArguments(cls, type, pending_types);
 
+  // Self referencing types may get finalized indirectly.
+  if (!type.IsFinalized()) {
+    // If the type is a function type, we also need to finalize the types in its
+    // signature, i.e. finalize the result type and parameter types of the
+    // signature function of this function type.
+    // We do this after marking this type as finalized in order to allow a
+    // typedef function type to refer to itself via its parameter types and
+    // result type.
+    if (type.IsFunctionType()) {
+      const Type& fun_type = Type::Cast(type);
+      const Class& scope_class = Class::Handle(zone, fun_type.type_class());
+      if (scope_class.IsTypedefClass()) {
+        Function& signature =
+            Function::Handle(zone, scope_class.signature_function());
+        if (!scope_class.is_type_finalized()) {
+          FinalizeSignature(scope_class, signature);
+        }
+        // If the function type is a generic typedef, instantiate its signature
+        // from its type arguments.
+        // Example: typedef T F<T>(T x) has uninstantiated signature (T x) => T.
+        // The instantiated signature of F(int) becomes (int x) => int.
+        // Note that after this step, the signature of the function type is not
+        // identical to the canonical signature of the typedef class anymore.
+        if (scope_class.IsGeneric() && !signature.HasInstantiatedSignature()) {
+          const TypeArguments& type_args =
+              TypeArguments::Handle(zone, fun_type.arguments());
+          if (FLAG_trace_type_finalization) {
+            THR_Print("Instantiating signature '%s' of typedef '%s'\n",
+                      String::Handle(zone, signature.Signature()).ToCString(),
+                      String::Handle(zone, fun_type.Name()).ToCString());
+          }
+          signature = signature.InstantiateSignatureFrom(type_args, Heap::kOld);
+          // Note that if type_args contains type parameters, signature is still
+          // uninstantiated here (typedef type parameters were substituted in
+          // the signature with typedef type arguments).
+        }
+        fun_type.set_signature(signature);
+      } else {
+        FinalizeSignature(cls, Function::Handle(zone, fun_type.signature()));
+      }
+    }
+
+    if (FLAG_trace_type_finalization) {
+      THR_Print("Marking type '%s' as finalized for class '%s'\n",
+                String::Handle(zone, type.Name()).ToCString(),
+                String::Handle(zone, cls.Name()).ToCString());
+    }
+    // Mark the type as finalized.
+    type.SetIsFinalized();
+  }
+
   // If we are done finalizing a graph of mutually recursive types, check their
   // bounds.
   if (is_root_type) {
     for (intptr_t i = pending_types->length() - 1; i >= 0; i--) {
       const AbstractType& type = pending_types->At(i);
       if (!type.IsMalformed() && !type.IsCanonical()) {
         CheckTypeBounds(cls, type);
       }
     }
   }
 
-  // If the type is a function type, we also need to finalize the types in its
-  // signature, i.e. finalize the result type and parameter types of the
-  // signature function of this function type.
-  // We do this after marking this type as finalized in order to allow a
-  // typedef function type to refer to itself via its parameter types and
-  // result type.
-  if (type.IsFunctionType()) {
-    ASSERT(!type.IsBeingFinalized());
-    const Type& fun_type = Type::Cast(type);
-    const Class& scope_class = Class::Handle(zone, fun_type.type_class());
-    if (scope_class.IsTypedefClass()) {
-      Function& signature =
-          Function::Handle(zone, scope_class.signature_function());
-      if (!scope_class.is_type_finalized()) {
-        FinalizeSignature(scope_class, signature);
-      }
-      // If the function type is a generic typedef, instantiate its signature
-      // from its type arguments.
-      // Example: typedef T F<T>(T x) has uninstantiated signature (T x) => T.
-      // The instantiated signature of F(int) becomes (int x) => int.
-      // Note that after this step, the signature of the function type is not
-      // identical to the canonical signature of the typedef class anymore.
-      if (scope_class.IsGeneric() && !signature.HasInstantiatedSignature()) {
-        const TypeArguments& type_args =
-            TypeArguments::Handle(zone, fun_type.arguments());
-        if (FLAG_trace_type_finalization) {
-          THR_Print("Instantiating signature '%s' of typedef '%s'\n",
-                    String::Handle(zone, signature.Signature()).ToCString(),
-                    String::Handle(zone, fun_type.Name()).ToCString());
-        }
-        signature = signature.InstantiateSignatureFrom(type_args, Heap::kOld);
-        // Note that if type_args contains type parameters, signature is still
-        // uninstantiated here (typedef type parameters were substituted in the
-        // signature with typedef type arguments).
-      }
-      fun_type.set_signature(signature);
-      // The type was already marked as finalized and uninstantiated in
-      // ExpandAndFinalizeTypeArguments above when its signature was not
-      // instantiated yet. Check again by calling ResetIsFinalized().
-      fun_type.ResetIsFinalized();
-    } else {
-      const Function& signature = Function::Handle(zone, fun_type.signature());
-      FinalizeSignature(cls, signature);
-    }
-  }
-
   if (FLAG_trace_type_finalization) {
     THR_Print("Done finalizing type '%s' with %" Pd " type args: %s\n",
               String::Handle(zone, type.Name()).ToCString(),
               num_expanded_type_arguments, type.ToCString());
   }
 
   if (finalization >= kCanonicalize) {
     if (FLAG_trace_type_finalization) {
       THR_Print("Canonicalizing type '%s'\n",
                 String::Handle(zone, type.Name()).ToCString());
@@ skipping 2458 matching lines @@
   ProgramVisitor::VisitFunctions(&function_visitor);
 
   class ClearCodeClassVisitor : public ClassVisitor {
     void Visit(const Class& cls) { cls.DisableAllocationStub(); }
   };
   ClearCodeClassVisitor class_visitor;
   ProgramVisitor::VisitClasses(&class_visitor);
 }
 
 }  // namespace dart