Use call counts to prevent cold calls from being inlined.

runtime/vm/flow_graph_inliner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/flow_graph_inliner.h"
 
 #include "vm/compiler.h"
 #include "vm/flags.h"
 #include "vm/flow_graph.h"
 #include "vm/flow_graph_builder.h"
@@ skipping 18 matching lines @@
 DEFINE_FLAG(int, inlining_in_loop_size_threshold, 80,
     "Inline functions in loops that have threshold or fewer instructions");
 DEFINE_FLAG(int, inlining_callee_call_sites_threshold, 1,
     "Always inline functions containing threshold or fewer calls.");
 DEFINE_FLAG(int, inlining_constant_arguments_count, 1,
     "Inline function calls with sufficient constant arguments "
     "and up to the increased threshold on instructions");
 DEFINE_FLAG(int, inlining_constant_arguments_size_threshold, 60,
     "Inline function calls with sufficient constant arguments "
     "and up to the increased threshold on instructions");
+DEFINE_FLAG(int, inlining_hotness, 10,
+    "Inline only hotter calls, in percents (0 .. 100); "
+    "default 10%: calls above-equal 10% of max-count are inlined.");
 
 DECLARE_FLAG(bool, print_flow_graph);
 DECLARE_FLAG(int, deoptimization_counter_threshold);
 DECLARE_FLAG(bool, verify_compiler);
 DECLARE_FLAG(bool, compiler_stats);
 
 #define TRACE_INLINING(statement)                                              \
   do {                                                                         \
     if (FLAG_trace_inlining) statement;                                        \
   } while (false)
@@ skipping 129 matching lines @@
 // A collection of call sites to consider for inlining.
 class CallSites : public FlowGraphVisitor {
  public:
   explicit CallSites(FlowGraph* flow_graph)
       : FlowGraphVisitor(flow_graph->postorder()),  // We don't use this order.
         static_calls_(),
         closure_calls_(),
         instance_calls_(),
         skip_static_call_deopt_ids_() { }
 
-  GrowableArray<StaticCallInstr*>* static_calls() {
-    return &static_calls_;
+  const GrowableArray<StaticCallInstr*>& static_calls() const {
+    return static_calls_;
   }
 
-  GrowableArray<ClosureCallInstr*>* closure_calls() {
-    return &closure_calls_;
+  const GrowableArray<ClosureCallInstr*>& closure_calls() const {
+    return closure_calls_;
   }
 
-  GrowableArray<PolymorphicInstanceCallInstr*>* instance_calls() {
-    return &instance_calls_;
+  struct InstanceCallInfo {
+    PolymorphicInstanceCallInstr* call;
+    double ratio;
+    explicit InstanceCallInfo(PolymorphicInstanceCallInstr* call_arg)
+        : call(call_arg), ratio(0.0) {}
+  };
+
+  const GrowableArray<InstanceCallInfo>& instance_calls() const {
+    return instance_calls_;
   }
 
   bool HasCalls() const {
     return !(static_calls_.is_empty() &&
              closure_calls_.is_empty() &&
              instance_calls_.is_empty());
   }
 
   void Clear() {
     static_calls_.Clear();
     closure_calls_.Clear();
     instance_calls_.Clear();
     skip_static_call_deopt_ids_.Clear();
   }
 
   void FindCallSites(FlowGraph* graph) {
     ASSERT(graph != NULL);
     const Function& function = graph->parsed_function().function();
     ASSERT(function.HasCode());
     const Code& code = Code::Handle(function.unoptimized_code());
+
     skip_static_call_deopt_ids_.Clear();
     code.ExtractUncalledStaticCallDeoptIds(&skip_static_call_deopt_ids_);
+
+    const intptr_t instance_call_start_ix = instance_calls_.length();
     for (BlockIterator block_it = graph->postorder_iterator();
          !block_it.Done();
          block_it.Advance()) {
       for (ForwardInstructionIterator it(block_it.Current());
            !it.Done();
            it.Advance()) {
         it.Current()->Accept(this);
       }
     }
+    // Compute instance call site ratio.
+    const intptr_t num_instance_calls =
+        instance_calls_.length() - instance_call_start_ix;
+    intptr_t max_count = 0;
+    GrowableArray<intptr_t> call_counts(num_instance_calls);
+    for (intptr_t i = 0; i < num_instance_calls; ++i) {
+      const intptr_t aggregate_count =
+          instance_calls_[i + instance_call_start_ix].
+              call-> ic_data().AggregateCount();

[Vyacheslav Egorov (Google), 2012/12/18 11:55:51]

remove space after ->

[srdjan, 2012/12/18 16:37:51]

Done.

+      call_counts.Add(aggregate_count);
+      if (aggregate_count > max_count) max_count = aggregate_count;
+    }
+
+
+    for (intptr_t i = 0; i < num_instance_calls; ++i) {
+      const double ratio = static_cast<double>(call_counts[i]) / max_count;
+      instance_calls_[i + instance_call_start_ix].ratio = ratio;
+    }
   }
 
   void VisitClosureCall(ClosureCallInstr* call) {
     closure_calls_.Add(call);
   }
 
   void VisitPolymorphicInstanceCall(PolymorphicInstanceCallInstr* call) {
-    instance_calls_.Add(call);
+    instance_calls_.Add(InstanceCallInfo(call));
   }
 
   void VisitStaticCall(StaticCallInstr* call) {
     if (!call->function().IsInlineable()) return;
     const intptr_t call_deopt_id = call->deopt_id();
     for (intptr_t i = 0; i < skip_static_call_deopt_ids_.length(); i++) {
       if (call_deopt_id == skip_static_call_deopt_ids_[i]) {
         // Do not inline this call.
         return;
       }
     }
     static_calls_.Add(call);
   }
 
  private:
   GrowableArray<StaticCallInstr*> static_calls_;
   GrowableArray<ClosureCallInstr*> closure_calls_;
-  GrowableArray<PolymorphicInstanceCallInstr*> instance_calls_;
+  GrowableArray<InstanceCallInfo> instance_calls_;
   GrowableArray<intptr_t> skip_static_call_deopt_ids_;
 
   DISALLOW_COPY_AND_ASSIGN(CallSites);
 };
 
 
 class CallSiteInliner : public ValueObject {
  public:
   explicit CallSiteInliner(FlowGraph* flow_graph)
       : caller_graph_(flow_graph),
@@ skipping 358 matching lines @@
     }
     *in_cache = false;
     ParsedFunction* parsed_function = new ParsedFunction(function);
     Parser::ParseFunction(parsed_function);
     parsed_function->AllocateVariables();
     return parsed_function;
   }
 
   void InlineStaticCalls() {
     const GrowableArray<StaticCallInstr*>& calls =
-        *inlining_call_sites_->static_calls();
+        inlining_call_sites_->static_calls();
     TRACE_INLINING(OS::Print("  Static Calls (%d)\n", calls.length()));
     for (intptr_t i = 0; i < calls.length(); ++i) {
       StaticCallInstr* call = calls[i];
       GrowableArray<Value*> arguments(call->ArgumentCount());
       for (int i = 0; i < call->ArgumentCount(); ++i) {
         arguments.Add(call->ArgumentAt(i)->value());
       }
       TryInlining(call->function(), call->argument_names(), &arguments, call);
     }
   }
 
   void InlineClosureCalls() {
     const GrowableArray<ClosureCallInstr*>& calls =
-        *inlining_call_sites_->closure_calls();
+        inlining_call_sites_->closure_calls();
     TRACE_INLINING(OS::Print("  Closure Calls (%d)\n", calls.length()));
     for (intptr_t i = 0; i < calls.length(); ++i) {
       ClosureCallInstr* call = calls[i];
       // Find the closure of the callee.
       ASSERT(call->ArgumentCount() > 0);
       const CreateClosureInstr* closure =
           call->ArgumentAt(0)->value()->definition()->AsCreateClosure();
       if (closure == NULL) {
         TRACE_INLINING(OS::Print("     Bailout: non-closure operator\n"));
         continue;
       }
       GrowableArray<Value*> arguments(call->ArgumentCount());
       for (int i = 0; i < call->ArgumentCount(); ++i) {
         arguments.Add(call->ArgumentAt(i)->value());
       }
       TryInlining(closure->function(),
                   call->argument_names(),
                   &arguments,
                   call);
     }
   }
 
   void InlineInstanceCalls() {
-    const GrowableArray<PolymorphicInstanceCallInstr*>& calls =
-        *inlining_call_sites_->instance_calls();
+    const GrowableArray<CallSites::InstanceCallInfo>& call_info =
+        inlining_call_sites_->instance_calls();
     TRACE_INLINING(OS::Print("  Polymorphic Instance Calls (%d)\n",
-                             calls.length()));
-    for (intptr_t i = 0; i < calls.length(); ++i) {
-      PolymorphicInstanceCallInstr* instr = calls[i];
+                             call_info.length()));
+    for (intptr_t i = 0; i < call_info.length(); ++i) {
+      PolymorphicInstanceCallInstr* instr = call_info[i].call;
       const ICData& ic_data = instr->ic_data();
       const Function& target = Function::ZoneHandle(ic_data.GetTargetAt(0));
       if (instr->with_checks()) {
         TRACE_INLINING(OS::Print(
-          "  => %s (deopt count %d)\n     Bailout: %"Pd" checks\n",
-          target.ToCString(),
-          target.deoptimization_counter(),
-          ic_data.NumberOfChecks()));
+            "  => %s (deopt count %d)\n     Bailout: %"Pd" checks\n",
+            target.ToCString(),
+            target.deoptimization_counter(),
+            ic_data.NumberOfChecks()));
+        continue;
+      }
+      if ((call_info[i].ratio * 100) < FLAG_inlining_hotness) {
+        TRACE_INLINING(OS::Print(
+            "  => %s (deopt count %d)\n     Bailout: cold %f\n",
+            target.ToCString(),
+            target.deoptimization_counter(),
+            call_info[i].ratio));
         continue;
       }
       GrowableArray<Value*> arguments(instr->ArgumentCount());
-      for (int i = 0; i < instr->ArgumentCount(); ++i) {
-        arguments.Add(instr->ArgumentAt(i)->value());
+      for (int arg_i = 0; arg_i < instr->ArgumentCount(); ++arg_i) {
+        arguments.Add(instr->ArgumentAt(arg_i)->value());
       }
       TryInlining(target,
                   instr->instance_call()->argument_names(),
                   &arguments,
                   instr);
     }
   }
 
   void AdjustForOptionalParameters(const ParsedFunction& parsed_function,
                                    const Array& argument_names,
@@ skipping 143 matching lines @@
         OS::Print("After Inlining of %s\n", flow_graph_->
                   parsed_function().function().ToFullyQualifiedCString());
         FlowGraphPrinter printer(*flow_graph_);
         printer.PrintBlocks();
       }
     }
   }
 }
 
 }  // namespace dart