| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/flow_graph_inliner.h" | 5 #include "vm/flow_graph_inliner.h" |
| 6 | 6 |
| 7 #include "vm/block_scheduler.h" | 7 #include "vm/block_scheduler.h" |
| 8 #include "vm/compiler.h" | 8 #include "vm/compiler.h" |
| 9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
| 10 #include "vm/flow_graph.h" | 10 #include "vm/flow_graph.h" |
| (...skipping 29 matching lines...) Expand all Loading... |
| 40 "Inline function calls with sufficient constant arguments " | 40 "Inline function calls with sufficient constant arguments " |
| 41 "and up to the increased threshold on instructions"); | 41 "and up to the increased threshold on instructions"); |
| 42 DEFINE_FLAG(int, inlining_constant_arguments_size_threshold, 60, | 42 DEFINE_FLAG(int, inlining_constant_arguments_size_threshold, 60, |
| 43 "Inline function calls with sufficient constant arguments " | 43 "Inline function calls with sufficient constant arguments " |
| 44 "and up to the increased threshold on instructions"); | 44 "and up to the increased threshold on instructions"); |
| 45 DEFINE_FLAG(int, inlining_hotness, 10, | 45 DEFINE_FLAG(int, inlining_hotness, 10, |
| 46 "Inline only hotter calls, in percents (0 .. 100); " | 46 "Inline only hotter calls, in percents (0 .. 100); " |
| 47 "default 10%: calls above-equal 10% of max-count are inlined."); | 47 "default 10%: calls above-equal 10% of max-count are inlined."); |
| 48 DEFINE_FLAG(bool, inline_recursive, true, | 48 DEFINE_FLAG(bool, inline_recursive, true, |
| 49 "Inline recursive calls."); | 49 "Inline recursive calls."); |
| 50 DEFINE_FLAG(bool, print_inline_tree, false, "Print inlining tree"); |
| 50 | 51 |
| 51 DECLARE_FLAG(bool, print_flow_graph); | 52 DECLARE_FLAG(bool, print_flow_graph); |
| 52 DECLARE_FLAG(bool, print_flow_graph_optimized); | 53 DECLARE_FLAG(bool, print_flow_graph_optimized); |
| 53 DECLARE_FLAG(int, deoptimization_counter_threshold); | 54 DECLARE_FLAG(int, deoptimization_counter_threshold); |
| 54 DECLARE_FLAG(bool, verify_compiler); | 55 DECLARE_FLAG(bool, verify_compiler); |
| 55 DECLARE_FLAG(bool, compiler_stats); | 56 DECLARE_FLAG(bool, compiler_stats); |
| 56 | 57 |
| 57 #define TRACE_INLINING(statement) \ | 58 #define TRACE_INLINING(statement) \ |
| 58 do { \ | 59 do { \ |
| 59 if (FLAG_trace_inlining) statement; \ | 60 if (FLAG_trace_inlining) statement; \ |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 154 | 155 |
| 155 | 156 |
| 156 // A collection of call sites to consider for inlining. | 157 // A collection of call sites to consider for inlining. |
| 157 class CallSites : public ValueObject { | 158 class CallSites : public ValueObject { |
| 158 public: | 159 public: |
| 159 explicit CallSites(FlowGraph* flow_graph) | 160 explicit CallSites(FlowGraph* flow_graph) |
| 160 : static_calls_(), | 161 : static_calls_(), |
| 161 closure_calls_(), | 162 closure_calls_(), |
| 162 instance_calls_() { } | 163 instance_calls_() { } |
| 163 | 164 |
| 164 const GrowableArray<ClosureCallInstr*>& closure_calls() const { | |
| 165 return closure_calls_; | |
| 166 } | |
| 167 | |
| 168 struct InstanceCallInfo { | 165 struct InstanceCallInfo { |
| 169 PolymorphicInstanceCallInstr* call; | 166 PolymorphicInstanceCallInstr* call; |
| 170 double ratio; | 167 double ratio; |
| 171 explicit InstanceCallInfo(PolymorphicInstanceCallInstr* call_arg) | 168 const Function* caller; |
| 172 : call(call_arg), ratio(0.0) {} | 169 InstanceCallInfo(PolymorphicInstanceCallInstr* call_arg, |
| 170 FlowGraph* flow_graph) |
| 171 : call(call_arg), |
| 172 ratio(0.0), |
| 173 caller(&flow_graph->parsed_function().function()) {} |
| 173 }; | 174 }; |
| 174 | 175 |
| 175 struct StaticCallInfo { | 176 struct StaticCallInfo { |
| 176 StaticCallInstr* call; | 177 StaticCallInstr* call; |
| 177 double ratio; | 178 double ratio; |
| 178 explicit StaticCallInfo(StaticCallInstr* value) | 179 const Function* caller; |
| 179 : call(value), ratio(0.0) {} | 180 StaticCallInfo(StaticCallInstr* value, FlowGraph* flow_graph) |
| 181 : call(value), |
| 182 ratio(0.0), |
| 183 caller(&flow_graph->parsed_function().function()) {} |
| 184 }; |
| 185 |
| 186 struct ClosureCallInfo { |
| 187 ClosureCallInstr* call; |
| 188 const Function* caller; |
| 189 ClosureCallInfo(ClosureCallInstr* value, FlowGraph* flow_graph) |
| 190 : call(value), |
| 191 caller(&flow_graph->parsed_function().function()) {} |
| 180 }; | 192 }; |
| 181 | 193 |
| 182 const GrowableArray<InstanceCallInfo>& instance_calls() const { | 194 const GrowableArray<InstanceCallInfo>& instance_calls() const { |
| 183 return instance_calls_; | 195 return instance_calls_; |
| 184 } | 196 } |
| 185 | 197 |
| 186 const GrowableArray<StaticCallInfo>& static_calls() const { | 198 const GrowableArray<StaticCallInfo>& static_calls() const { |
| 187 return static_calls_; | 199 return static_calls_; |
| 188 } | 200 } |
| 189 | 201 |
| 202 const GrowableArray<ClosureCallInfo>& closure_calls() const { |
| 203 return closure_calls_; |
| 204 } |
| 205 |
| 190 bool HasCalls() const { | 206 bool HasCalls() const { |
| 191 return !(static_calls_.is_empty() && | 207 return !(static_calls_.is_empty() && |
| 192 closure_calls_.is_empty() && | 208 closure_calls_.is_empty() && |
| 193 instance_calls_.is_empty()); | 209 instance_calls_.is_empty()); |
| 194 } | 210 } |
| 195 | 211 |
| 196 void Clear() { | 212 void Clear() { |
| 197 static_calls_.Clear(); | 213 static_calls_.Clear(); |
| 198 closure_calls_.Clear(); | 214 closure_calls_.Clear(); |
| 199 instance_calls_.Clear(); | 215 instance_calls_.Clear(); |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 254 !block_it.Done(); | 270 !block_it.Done(); |
| 255 block_it.Advance()) { | 271 block_it.Advance()) { |
| 256 for (ForwardInstructionIterator it(block_it.Current()); | 272 for (ForwardInstructionIterator it(block_it.Current()); |
| 257 !it.Done(); | 273 !it.Done(); |
| 258 it.Advance()) { | 274 it.Advance()) { |
| 259 Instruction* current = it.Current(); | 275 Instruction* current = it.Current(); |
| 260 if (only_recognized_methods) { | 276 if (only_recognized_methods) { |
| 261 PolymorphicInstanceCallInstr* instance_call = | 277 PolymorphicInstanceCallInstr* instance_call = |
| 262 current->AsPolymorphicInstanceCall(); | 278 current->AsPolymorphicInstanceCall(); |
| 263 if ((instance_call != NULL) && instance_call->HasRecognizedTarget()) { | 279 if ((instance_call != NULL) && instance_call->HasRecognizedTarget()) { |
| 264 instance_calls_.Add(InstanceCallInfo(instance_call)); | 280 instance_calls_.Add(InstanceCallInfo(instance_call, graph)); |
| 265 } | 281 } |
| 266 continue; | 282 continue; |
| 267 } | 283 } |
| 268 // Collect all call sites (!only_recognized_methods). | 284 // Collect all call sites (!only_recognized_methods). |
| 269 ClosureCallInstr* closure_call = current->AsClosureCall(); | 285 ClosureCallInstr* closure_call = current->AsClosureCall(); |
| 270 if (closure_call != NULL) { | 286 if (closure_call != NULL) { |
| 271 closure_calls_.Add(closure_call); | 287 closure_calls_.Add(ClosureCallInfo(closure_call, graph)); |
| 272 continue; | 288 continue; |
| 273 } | 289 } |
| 274 StaticCallInstr* static_call = current->AsStaticCall(); | 290 StaticCallInstr* static_call = current->AsStaticCall(); |
| 275 if (static_call != NULL) { | 291 if (static_call != NULL) { |
| 276 static_calls_.Add(StaticCallInfo(static_call)); | 292 static_calls_.Add(StaticCallInfo(static_call, graph)); |
| 277 continue; | 293 continue; |
| 278 } | 294 } |
| 279 PolymorphicInstanceCallInstr* instance_call = | 295 PolymorphicInstanceCallInstr* instance_call = |
| 280 current->AsPolymorphicInstanceCall(); | 296 current->AsPolymorphicInstanceCall(); |
| 281 if (instance_call != NULL) { | 297 if (instance_call != NULL) { |
| 282 instance_calls_.Add(InstanceCallInfo(instance_call)); | 298 instance_calls_.Add(InstanceCallInfo(instance_call, graph)); |
| 283 continue; | 299 continue; |
| 284 } | 300 } |
| 285 } | 301 } |
| 286 } | 302 } |
| 287 ComputeCallSiteRatio(static_call_start_ix, instance_call_start_ix); | 303 ComputeCallSiteRatio(static_call_start_ix, instance_call_start_ix); |
| 288 } | 304 } |
| 289 | 305 |
| 290 private: | 306 private: |
| 291 GrowableArray<StaticCallInfo> static_calls_; | 307 GrowableArray<StaticCallInfo> static_calls_; |
| 292 GrowableArray<ClosureCallInstr*> closure_calls_; | 308 GrowableArray<ClosureCallInfo> closure_calls_; |
| 293 GrowableArray<InstanceCallInfo> instance_calls_; | 309 GrowableArray<InstanceCallInfo> instance_calls_; |
| 294 | 310 |
| 295 DISALLOW_COPY_AND_ASSIGN(CallSites); | 311 DISALLOW_COPY_AND_ASSIGN(CallSites); |
| 296 }; | 312 }; |
| 297 | 313 |
| 298 | 314 |
| 299 struct InlinedCallData { | 315 struct InlinedCallData { |
| 300 InlinedCallData(Definition* call, GrowableArray<Value*>* arguments) | 316 InlinedCallData(Definition* call, |
| 317 GrowableArray<Value*>* arguments, |
| 318 const Function& caller) |
| 301 : call(call), | 319 : call(call), |
| 302 arguments(arguments), | 320 arguments(arguments), |
| 303 callee_graph(NULL), | 321 callee_graph(NULL), |
| 304 parameter_stubs(NULL), | 322 parameter_stubs(NULL), |
| 305 exit_collector(NULL) { } | 323 exit_collector(NULL), |
| 324 caller_(caller) { } |
| 306 | 325 |
| 307 Definition* call; | 326 Definition* call; |
| 308 GrowableArray<Value*>* arguments; | 327 GrowableArray<Value*>* arguments; |
| 309 FlowGraph* callee_graph; | 328 FlowGraph* callee_graph; |
| 310 ZoneGrowableArray<Definition*>* parameter_stubs; | 329 ZoneGrowableArray<Definition*>* parameter_stubs; |
| 311 InlineExitCollector* exit_collector; | 330 InlineExitCollector* exit_collector; |
| 331 const Function& caller_; |
| 332 }; |
| 333 |
| 334 |
| 335 // Structure for collecting inline data needed to print inlining tree. |
| 336 struct InlinedInfo { |
| 337 const Function* caller; |
| 338 const Function* inlined; |
| 339 intptr_t inlined_depth; |
| 340 const Definition* call_instr; |
| 341 InlinedInfo(const Function* caller_function, |
| 342 const Function* inlined_function, |
| 343 const intptr_t depth, |
| 344 const Definition* call) |
| 345 : caller(caller_function), |
| 346 inlined(inlined_function), |
| 347 inlined_depth(depth), |
| 348 call_instr(call) {} |
| 312 }; | 349 }; |
| 313 | 350 |
| 314 | 351 |
| 315 class CallSiteInliner; | 352 class CallSiteInliner; |
| 316 | 353 |
| 317 class PolymorphicInliner : public ValueObject { | 354 class PolymorphicInliner : public ValueObject { |
| 318 public: | 355 public: |
| 319 PolymorphicInliner(CallSiteInliner* owner, | 356 PolymorphicInliner(CallSiteInliner* owner, |
| 320 PolymorphicInstanceCallInstr* call); | 357 PolymorphicInstanceCallInstr* call, |
| 358 const Function& caller_function); |
| 321 | 359 |
| 322 void Inline(); | 360 void Inline(); |
| 323 | 361 |
| 324 private: | 362 private: |
| 325 bool CheckInlinedDuplicate(const Function& target); | 363 bool CheckInlinedDuplicate(const Function& target); |
| 326 bool CheckNonInlinedDuplicate(const Function& target); | 364 bool CheckNonInlinedDuplicate(const Function& target); |
| 327 | 365 |
| 328 bool TryInlining(intptr_t receiver_cid, const Function& target); | 366 bool TryInliningPoly(intptr_t receiver_cid, const Function& target); |
| 329 bool TryInlineRecognizedMethod(intptr_t receiver_cid, const Function& target); | 367 bool TryInlineRecognizedMethod(intptr_t receiver_cid, const Function& target); |
| 330 | 368 |
| 331 TargetEntryInstr* BuildDecisionGraph(); | 369 TargetEntryInstr* BuildDecisionGraph(); |
| 332 | 370 |
| 333 CallSiteInliner* const owner_; | 371 CallSiteInliner* const owner_; |
| 334 PolymorphicInstanceCallInstr* const call_; | 372 PolymorphicInstanceCallInstr* const call_; |
| 335 const intptr_t num_variants_; | 373 const intptr_t num_variants_; |
| 336 GrowableArray<CidTarget> variants_; | 374 GrowableArray<CidTarget> variants_; |
| 337 | 375 |
| 338 GrowableArray<CidTarget> inlined_variants_; | 376 GrowableArray<CidTarget> inlined_variants_; |
| 339 GrowableArray<CidTarget> non_inlined_variants_; | 377 GrowableArray<CidTarget> non_inlined_variants_; |
| 340 GrowableArray<BlockEntryInstr*> inlined_entries_; | 378 GrowableArray<BlockEntryInstr*> inlined_entries_; |
| 341 InlineExitCollector* exit_collector_; | 379 InlineExitCollector* exit_collector_; |
| 380 |
| 381 const Function& caller_function_; |
| 342 }; | 382 }; |
| 343 | 383 |
| 344 | 384 |
| 345 class CallSiteInliner : public ValueObject { | 385 class CallSiteInliner : public ValueObject { |
| 346 public: | 386 public: |
| 347 explicit CallSiteInliner(FlowGraph* flow_graph) | 387 explicit CallSiteInliner(FlowGraph* flow_graph) |
| 348 : caller_graph_(flow_graph), | 388 : caller_graph_(flow_graph), |
| 349 inlined_(false), | 389 inlined_(false), |
| 350 initial_size_(flow_graph->InstructionCount()), | 390 initial_size_(flow_graph->InstructionCount()), |
| 351 inlined_size_(0), | 391 inlined_size_(0), |
| 352 inlining_depth_(1), | 392 inlining_depth_(1), |
| 353 collected_call_sites_(NULL), | 393 collected_call_sites_(NULL), |
| 354 inlining_call_sites_(NULL), | 394 inlining_call_sites_(NULL), |
| 355 function_cache_() { } | 395 function_cache_(), |
| 396 inlined_info_() { } |
| 356 | 397 |
| 357 FlowGraph* caller_graph() const { return caller_graph_; } | 398 FlowGraph* caller_graph() const { return caller_graph_; } |
| 358 | 399 |
| 359 // Inlining heuristics based on Cooper et al. 2008. | 400 // Inlining heuristics based on Cooper et al. 2008. |
| 360 bool ShouldWeInline(const Function& callee, | 401 bool ShouldWeInline(const Function& callee, |
| 361 intptr_t instr_count, | 402 intptr_t instr_count, |
| 362 intptr_t call_site_count, | 403 intptr_t call_site_count, |
| 363 intptr_t const_arg_count) { | 404 intptr_t const_arg_count) { |
| 364 if (inlined_size_ > FLAG_inlining_caller_size_threshold) { | 405 if (inlined_size_ > FLAG_inlining_caller_size_threshold) { |
| 365 // Prevent methods becoming humongous and thus slow to compile. | 406 // Prevent methods becoming humongous and thus slow to compile. |
| (...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 640 // list of guarded fields. | 681 // list of guarded fields. |
| 641 for (intptr_t i = 0; i < callee_graph->guarded_fields()->length(); ++i) { | 682 for (intptr_t i = 0; i < callee_graph->guarded_fields()->length(); ++i) { |
| 642 FlowGraph::AddToGuardedFields(caller_graph_->guarded_fields(), | 683 FlowGraph::AddToGuardedFields(caller_graph_->guarded_fields(), |
| 643 (*callee_graph->guarded_fields())[i]); | 684 (*callee_graph->guarded_fields())[i]); |
| 644 } | 685 } |
| 645 | 686 |
| 646 // We allocate a ZoneHandle for the unoptimized code so that it cannot be | 687 // We allocate a ZoneHandle for the unoptimized code so that it cannot be |
| 647 // disconnected from its function during the rest of compilation. | 688 // disconnected from its function during the rest of compilation. |
| 648 Code::ZoneHandle(unoptimized_code.raw()); | 689 Code::ZoneHandle(unoptimized_code.raw()); |
| 649 TRACE_INLINING(OS::Print(" Success\n")); | 690 TRACE_INLINING(OS::Print(" Success\n")); |
| 691 if (FLAG_print_inline_tree) { |
| 692 inlined_info_.Add( |
| 693 InlinedInfo(&call_data->caller_, &function, inlining_depth_, call)); |
| 694 } |
| 650 return true; | 695 return true; |
| 651 } else { | 696 } else { |
| 652 Error& error = Error::Handle(); | 697 Error& error = Error::Handle(); |
| 653 error = isolate->object_store()->sticky_error(); | 698 error = isolate->object_store()->sticky_error(); |
| 654 isolate->object_store()->clear_sticky_error(); | 699 isolate->object_store()->clear_sticky_error(); |
| 655 isolate->set_deopt_id(prev_deopt_id); | 700 isolate->set_deopt_id(prev_deopt_id); |
| 656 TRACE_INLINING(OS::Print(" Bailout: %s\n", error.ToErrorCString())); | 701 TRACE_INLINING(OS::Print(" Bailout: %s\n", error.ToErrorCString())); |
| 657 return false; | 702 return false; |
| 658 } | 703 } |
| 659 } | 704 } |
| 660 | 705 |
| 706 void PrintInlinedInfo(const Function& top) { |
| 707 OS::Print("Inlining into: %s\n", top.ToFullyQualifiedCString()); |
| 708 PrintInlinedInfoFor(top, 1); |
| 709 } |
| 710 |
| 661 private: | 711 private: |
| 662 friend class PolymorphicInliner; | 712 friend class PolymorphicInliner; |
| 663 | 713 |
| 714 void PrintInlinedInfoFor(const Function& caller, intptr_t depth) { |
| 715 for (intptr_t i = 0; i < inlined_info_.length(); i++) { |
| 716 const InlinedInfo& info = inlined_info_[i]; |
| 717 if ((info.inlined_depth == depth) && |
| 718 (info.caller->raw() == caller.raw())) { |
| 719 for (int t = 0; t < depth; t++) { |
| 720 OS::Print(" "); |
| 721 } |
| 722 OS::Print("%" Pd " %s\n", |
| 723 info.call_instr->GetDeoptId(), |
| 724 info.inlined->ToQualifiedCString()); |
| 725 PrintInlinedInfoFor(*info.inlined, depth + 1); |
| 726 } |
| 727 } |
| 728 } |
| 729 |
| 664 void InlineCall(InlinedCallData* call_data) { | 730 void InlineCall(InlinedCallData* call_data) { |
| 665 TimerScope timer(FLAG_compiler_stats, | 731 TimerScope timer(FLAG_compiler_stats, |
| 666 &CompilerStats::graphinliner_subst_timer, | 732 &CompilerStats::graphinliner_subst_timer, |
| 667 Isolate::Current()); | 733 Isolate::Current()); |
| 668 | 734 |
| 669 // For closure calls: Store context value. | 735 // For closure calls: Store context value. |
| 670 FlowGraph* callee_graph = call_data->callee_graph; | 736 FlowGraph* callee_graph = call_data->callee_graph; |
| 671 TargetEntryInstr* callee_entry = | 737 TargetEntryInstr* callee_entry = |
| 672 callee_graph->graph_entry()->normal_entry(); | 738 callee_graph->graph_entry()->normal_entry(); |
| 673 ClosureCallInstr* closure_call = call_data->call->AsClosureCall(); | 739 ClosureCallInstr* closure_call = call_data->call->AsClosureCall(); |
| (...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 776 " => %s (deopt count %d)\n Bailout: cold %f\n", | 842 " => %s (deopt count %d)\n Bailout: cold %f\n", |
| 777 target.ToCString(), | 843 target.ToCString(), |
| 778 target.deoptimization_counter(), | 844 target.deoptimization_counter(), |
| 779 call_info[call_idx].ratio)); | 845 call_info[call_idx].ratio)); |
| 780 continue; | 846 continue; |
| 781 } | 847 } |
| 782 GrowableArray<Value*> arguments(call->ArgumentCount()); | 848 GrowableArray<Value*> arguments(call->ArgumentCount()); |
| 783 for (int i = 0; i < call->ArgumentCount(); ++i) { | 849 for (int i = 0; i < call->ArgumentCount(); ++i) { |
| 784 arguments.Add(call->PushArgumentAt(i)->value()); | 850 arguments.Add(call->PushArgumentAt(i)->value()); |
| 785 } | 851 } |
| 786 InlinedCallData call_data(call, &arguments); | 852 InlinedCallData call_data(call, &arguments, *call_info[call_idx].caller); |
| 787 if (TryInlining(call->function(), call->argument_names(), &call_data)) { | 853 if (TryInlining(call->function(), call->argument_names(), &call_data)) { |
| 788 InlineCall(&call_data); | 854 InlineCall(&call_data); |
| 789 } | 855 } |
| 790 } | 856 } |
| 791 } | 857 } |
| 792 | 858 |
| 793 void InlineClosureCalls() { | 859 void InlineClosureCalls() { |
| 794 const GrowableArray<ClosureCallInstr*>& calls = | 860 const GrowableArray<CallSites::ClosureCallInfo>& call_info = |
| 795 inlining_call_sites_->closure_calls(); | 861 inlining_call_sites_->closure_calls(); |
| 796 TRACE_INLINING(OS::Print(" Closure Calls (%" Pd ")\n", calls.length())); | 862 TRACE_INLINING(OS::Print(" Closure Calls (%" Pd ")\n", |
| 797 for (intptr_t i = 0; i < calls.length(); ++i) { | 863 call_info.length())); |
| 798 ClosureCallInstr* call = calls[i]; | 864 for (intptr_t call_idx = 0; call_idx < call_info.length(); ++call_idx) { |
| 865 ClosureCallInstr* call = call_info[call_idx].call; |
| 799 // Find the closure of the callee. | 866 // Find the closure of the callee. |
| 800 ASSERT(call->ArgumentCount() > 0); | 867 ASSERT(call->ArgumentCount() > 0); |
| 801 Function& target = Function::ZoneHandle(); | 868 Function& target = Function::ZoneHandle(); |
| 802 AllocateObjectInstr* alloc = | 869 AllocateObjectInstr* alloc = |
| 803 call->ArgumentAt(0)->AsAllocateObject(); | 870 call->ArgumentAt(0)->AsAllocateObject(); |
| 804 if ((alloc != NULL) && !alloc->closure_function().IsNull()) { | 871 if ((alloc != NULL) && !alloc->closure_function().IsNull()) { |
| 805 target ^= alloc->closure_function().raw(); | 872 target ^= alloc->closure_function().raw(); |
| 806 ASSERT(target.signature_class() == alloc->cls().raw()); | 873 ASSERT(target.signature_class() == alloc->cls().raw()); |
| 807 } | 874 } |
| 808 if (target.IsNull()) { | 875 if (target.IsNull()) { |
| 809 TRACE_INLINING(OS::Print(" Bailout: non-closure operator\n")); | 876 TRACE_INLINING(OS::Print(" Bailout: non-closure operator\n")); |
| 810 continue; | 877 continue; |
| 811 } | 878 } |
| 812 GrowableArray<Value*> arguments(call->ArgumentCount()); | 879 GrowableArray<Value*> arguments(call->ArgumentCount()); |
| 813 for (int i = 0; i < call->ArgumentCount(); ++i) { | 880 for (int i = 0; i < call->ArgumentCount(); ++i) { |
| 814 arguments.Add(call->PushArgumentAt(i)->value()); | 881 arguments.Add(call->PushArgumentAt(i)->value()); |
| 815 } | 882 } |
| 816 InlinedCallData call_data(call, &arguments); | 883 InlinedCallData call_data(call, &arguments, *call_info[call_idx].caller); |
| 817 if (TryInlining(target, | 884 if (TryInlining(target, |
| 818 call->argument_names(), | 885 call->argument_names(), |
| 819 &call_data)) { | 886 &call_data)) { |
| 820 InlineCall(&call_data); | 887 InlineCall(&call_data); |
| 821 } | 888 } |
| 822 } | 889 } |
| 823 } | 890 } |
| 824 | 891 |
| 825 void InlineInstanceCalls() { | 892 void InlineInstanceCalls() { |
| 826 const GrowableArray<CallSites::InstanceCallInfo>& call_info = | 893 const GrowableArray<CallSites::InstanceCallInfo>& call_info = |
| 827 inlining_call_sites_->instance_calls(); | 894 inlining_call_sites_->instance_calls(); |
| 828 TRACE_INLINING(OS::Print(" Polymorphic Instance Calls (%" Pd ")\n", | 895 TRACE_INLINING(OS::Print(" Polymorphic Instance Calls (%" Pd ")\n", |
| 829 call_info.length())); | 896 call_info.length())); |
| 830 for (intptr_t call_idx = 0; call_idx < call_info.length(); ++call_idx) { | 897 for (intptr_t call_idx = 0; call_idx < call_info.length(); ++call_idx) { |
| 831 PolymorphicInstanceCallInstr* call = call_info[call_idx].call; | 898 PolymorphicInstanceCallInstr* call = call_info[call_idx].call; |
| 832 if (call->with_checks()) { | 899 if (call->with_checks()) { |
| 833 PolymorphicInliner inliner(this, call); | 900 const Function& cl = *call_info[call_idx].caller; |
| 901 PolymorphicInliner inliner(this, call, cl); |
| 834 inliner.Inline(); | 902 inliner.Inline(); |
| 835 continue; | 903 continue; |
| 836 } | 904 } |
| 837 | 905 |
| 838 const ICData& ic_data = call->ic_data(); | 906 const ICData& ic_data = call->ic_data(); |
| 839 const Function& target = Function::ZoneHandle(ic_data.GetTargetAt(0)); | 907 const Function& target = Function::ZoneHandle(ic_data.GetTargetAt(0)); |
| 840 if (!FlowGraphInliner::AlwaysInline(target) && | 908 if (!FlowGraphInliner::AlwaysInline(target) && |
| 841 (call_info[call_idx].ratio * 100) < FLAG_inlining_hotness) { | 909 (call_info[call_idx].ratio * 100) < FLAG_inlining_hotness) { |
| 842 TRACE_INLINING(OS::Print( | 910 TRACE_INLINING(OS::Print( |
| 843 " => %s (deopt count %d)\n Bailout: cold %f\n", | 911 " => %s (deopt count %d)\n Bailout: cold %f\n", |
| 844 target.ToCString(), | 912 target.ToCString(), |
| 845 target.deoptimization_counter(), | 913 target.deoptimization_counter(), |
| 846 call_info[call_idx].ratio)); | 914 call_info[call_idx].ratio)); |
| 847 continue; | 915 continue; |
| 848 } | 916 } |
| 849 GrowableArray<Value*> arguments(call->ArgumentCount()); | 917 GrowableArray<Value*> arguments(call->ArgumentCount()); |
| 850 for (int arg_i = 0; arg_i < call->ArgumentCount(); ++arg_i) { | 918 for (int arg_i = 0; arg_i < call->ArgumentCount(); ++arg_i) { |
| 851 arguments.Add(call->PushArgumentAt(arg_i)->value()); | 919 arguments.Add(call->PushArgumentAt(arg_i)->value()); |
| 852 } | 920 } |
| 853 InlinedCallData call_data(call, &arguments); | 921 InlinedCallData call_data(call, &arguments, *call_info[call_idx].caller); |
| 854 if (TryInlining(target, | 922 if (TryInlining(target, |
| 855 call->instance_call()->argument_names(), | 923 call->instance_call()->argument_names(), |
| 856 &call_data)) { | 924 &call_data)) { |
| 857 InlineCall(&call_data); | 925 InlineCall(&call_data); |
| 858 } | 926 } |
| 859 } | 927 } |
| 860 } | 928 } |
| 861 | 929 |
| 862 bool AdjustForOptionalParameters(const ParsedFunction& parsed_function, | 930 bool AdjustForOptionalParameters(const ParsedFunction& parsed_function, |
| 863 const Array& argument_names, | 931 const Array& argument_names, |
| (...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 944 if (arg != NULL) { | 1012 if (arg != NULL) { |
| 945 param_stubs->Add(CreateParameterStub(i, arg, callee_graph)); | 1013 param_stubs->Add(CreateParameterStub(i, arg, callee_graph)); |
| 946 } else { | 1014 } else { |
| 947 param_stubs->Add( | 1015 param_stubs->Add( |
| 948 GetDefaultValue(i - fixed_param_count, parsed_function)); | 1016 GetDefaultValue(i - fixed_param_count, parsed_function)); |
| 949 } | 1017 } |
| 950 } | 1018 } |
| 951 return argument_names_count == match_count; | 1019 return argument_names_count == match_count; |
| 952 } | 1020 } |
| 953 | 1021 |
| 954 | |
| 955 FlowGraph* caller_graph_; | 1022 FlowGraph* caller_graph_; |
| 956 bool inlined_; | 1023 bool inlined_; |
| 957 intptr_t initial_size_; | 1024 intptr_t initial_size_; |
| 958 intptr_t inlined_size_; | 1025 intptr_t inlined_size_; |
| 959 intptr_t inlining_depth_; | 1026 intptr_t inlining_depth_; |
| 960 CallSites* collected_call_sites_; | 1027 CallSites* collected_call_sites_; |
| 961 CallSites* inlining_call_sites_; | 1028 CallSites* inlining_call_sites_; |
| 962 GrowableArray<ParsedFunction*> function_cache_; | 1029 GrowableArray<ParsedFunction*> function_cache_; |
| 1030 GrowableArray<InlinedInfo> inlined_info_; |
| 963 | 1031 |
| 964 DISALLOW_COPY_AND_ASSIGN(CallSiteInliner); | 1032 DISALLOW_COPY_AND_ASSIGN(CallSiteInliner); |
| 965 }; | 1033 }; |
| 966 | 1034 |
| 967 | 1035 |
| 968 PolymorphicInliner::PolymorphicInliner(CallSiteInliner* owner, | 1036 PolymorphicInliner::PolymorphicInliner(CallSiteInliner* owner, |
| 969 PolymorphicInstanceCallInstr* call) | 1037 PolymorphicInstanceCallInstr* call, |
| 1038 const Function& caller_function) |
| 970 : owner_(owner), | 1039 : owner_(owner), |
| 971 call_(call), | 1040 call_(call), |
| 972 num_variants_(call->ic_data().NumberOfChecks()), | 1041 num_variants_(call->ic_data().NumberOfChecks()), |
| 973 variants_(num_variants_), | 1042 variants_(num_variants_), |
| 974 inlined_variants_(num_variants_), | 1043 inlined_variants_(num_variants_), |
| 975 non_inlined_variants_(num_variants_), | 1044 non_inlined_variants_(num_variants_), |
| 976 inlined_entries_(num_variants_), | 1045 inlined_entries_(num_variants_), |
| 977 exit_collector_(new InlineExitCollector(owner->caller_graph(), call)) { | 1046 exit_collector_(new InlineExitCollector(owner->caller_graph(), call)), |
| 1047 caller_function_(caller_function) { |
| 978 } | 1048 } |
| 979 | 1049 |
| 980 | 1050 |
| 981 // Inlined bodies are shared if two different class ids have the same | 1051 // Inlined bodies are shared if two different class ids have the same |
| 982 // inlined target. This sharing is represented by using three different | 1052 // inlined target. This sharing is represented by using three different |
| 983 // types of entries in the inlined_entries_ array: | 1053 // types of entries in the inlined_entries_ array: |
| 984 // | 1054 // |
| 985 // * GraphEntry: the inlined body is not shared. | 1055 // * GraphEntry: the inlined body is not shared. |
| 986 // | 1056 // |
| 987 // * TargetEntry: the inlined body is shared and this is the first variant. | 1057 // * TargetEntry: the inlined body is shared and this is the first variant. |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1043 for (intptr_t i = 0; i < non_inlined_variants_.length(); ++i) { | 1113 for (intptr_t i = 0; i < non_inlined_variants_.length(); ++i) { |
| 1044 if (target.raw() == non_inlined_variants_[i].target->raw()) { | 1114 if (target.raw() == non_inlined_variants_[i].target->raw()) { |
| 1045 return true; | 1115 return true; |
| 1046 } | 1116 } |
| 1047 } | 1117 } |
| 1048 | 1118 |
| 1049 return false; | 1119 return false; |
| 1050 } | 1120 } |
| 1051 | 1121 |
| 1052 | 1122 |
| 1053 bool PolymorphicInliner::TryInlining(intptr_t receiver_cid, | 1123 bool PolymorphicInliner::TryInliningPoly(intptr_t receiver_cid, |
| 1054 const Function& target) { | 1124 const Function& target) { |
| 1055 if (!target.IsInlineable()) { | 1125 if (!target.IsInlineable()) { |
| 1056 if (TryInlineRecognizedMethod(receiver_cid, target)) { | 1126 if (TryInlineRecognizedMethod(receiver_cid, target)) { |
| 1057 owner_->inlined_ = true; | 1127 owner_->inlined_ = true; |
| 1058 return true; | 1128 return true; |
| 1059 } | 1129 } |
| 1060 return false; | 1130 return false; |
| 1061 } | 1131 } |
| 1062 | 1132 |
| 1063 GrowableArray<Value*> arguments(call_->ArgumentCount()); | 1133 GrowableArray<Value*> arguments(call_->ArgumentCount()); |
| 1064 for (int i = 0; i < call_->ArgumentCount(); ++i) { | 1134 for (int i = 0; i < call_->ArgumentCount(); ++i) { |
| 1065 arguments.Add(call_->PushArgumentAt(i)->value()); | 1135 arguments.Add(call_->PushArgumentAt(i)->value()); |
| 1066 } | 1136 } |
| 1067 InlinedCallData call_data(call_, &arguments); | 1137 InlinedCallData call_data(call_, &arguments, caller_function_); |
| 1068 if (!owner_->TryInlining(target, | 1138 if (!owner_->TryInlining(target, |
| 1069 call_->instance_call()->argument_names(), | 1139 call_->instance_call()->argument_names(), |
| 1070 &call_data)) { | 1140 &call_data)) { |
| 1071 return false; | 1141 return false; |
| 1072 } | 1142 } |
| 1073 | 1143 |
| 1074 FlowGraph* callee_graph = call_data.callee_graph; | 1144 FlowGraph* callee_graph = call_data.callee_graph; |
| 1075 call_data.exit_collector->PrepareGraphs(callee_graph); | 1145 call_data.exit_collector->PrepareGraphs(callee_graph); |
| 1076 inlined_entries_.Add(callee_graph->graph_entry()); | 1146 inlined_entries_.Add(callee_graph->graph_entry()); |
| 1077 exit_collector_->Union(call_data.exit_collector); | 1147 exit_collector_->Union(call_data.exit_collector); |
| (...skipping 319 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1397 | 1467 |
| 1398 // Also check if this is the same target as an earlier non-inlined | 1468 // Also check if this is the same target as an earlier non-inlined |
| 1399 // variant. If so and since inlining decisions are costly, do not try | 1469 // variant. If so and since inlining decisions are costly, do not try |
| 1400 // to inline this variant. | 1470 // to inline this variant. |
| 1401 if (CheckNonInlinedDuplicate(target)) { | 1471 if (CheckNonInlinedDuplicate(target)) { |
| 1402 non_inlined_variants_.Add(variants_[var_idx]); | 1472 non_inlined_variants_.Add(variants_[var_idx]); |
| 1403 continue; | 1473 continue; |
| 1404 } | 1474 } |
| 1405 | 1475 |
| 1406 // Make an inlining decision. | 1476 // Make an inlining decision. |
| 1407 if (TryInlining(receiver_cid, target)) { | 1477 if (TryInliningPoly(receiver_cid, target)) { |
| 1408 inlined_variants_.Add(variants_[var_idx]); | 1478 inlined_variants_.Add(variants_[var_idx]); |
| 1409 } else { | 1479 } else { |
| 1410 non_inlined_variants_.Add(variants_[var_idx]); | 1480 non_inlined_variants_.Add(variants_[var_idx]); |
| 1411 } | 1481 } |
| 1412 } | 1482 } |
| 1413 | 1483 |
| 1414 // If there are no inlined variants, leave the call in place. | 1484 // If there are no inlined variants, leave the call in place. |
| 1415 if (inlined_variants_.is_empty()) return; | 1485 if (inlined_variants_.is_empty()) return; |
| 1416 | 1486 |
| 1417 // Now build a decision tree (a DAG because of shared inline variants) and | 1487 // Now build a decision tree (a DAG because of shared inline variants) and |
| (...skipping 28 matching lines...) Expand all Loading... |
| 1446 } | 1516 } |
| 1447 return MethodRecognizer::AlwaysInline(function); | 1517 return MethodRecognizer::AlwaysInline(function); |
| 1448 } | 1518 } |
| 1449 | 1519 |
| 1450 | 1520 |
| 1451 void FlowGraphInliner::Inline() { | 1521 void FlowGraphInliner::Inline() { |
| 1452 // Collect graph info and store it on the function. | 1522 // Collect graph info and store it on the function. |
| 1453 // We might later use it for an early bailout from the inlining. | 1523 // We might later use it for an early bailout from the inlining. |
| 1454 CollectGraphInfo(flow_graph_); | 1524 CollectGraphInfo(flow_graph_); |
| 1455 | 1525 |
| 1526 const Function& top = flow_graph_->parsed_function().function(); |
| 1456 if ((FLAG_inlining_filter != NULL) && | 1527 if ((FLAG_inlining_filter != NULL) && |
| 1457 (strstr(flow_graph_-> | 1528 (strstr(top.ToFullyQualifiedCString(), FLAG_inlining_filter) == NULL)) { |
| 1458 parsed_function().function().ToFullyQualifiedCString(), | |
| 1459 FLAG_inlining_filter) == NULL)) { | |
| 1460 return; | 1529 return; |
| 1461 } | 1530 } |
| 1462 | 1531 |
| 1463 TRACE_INLINING(OS::Print( | 1532 TRACE_INLINING(OS::Print("Inlining calls in %s\n", top.ToCString())); |
| 1464 "Inlining calls in %s\n", | |
| 1465 flow_graph_->parsed_function().function().ToCString())); | |
| 1466 | 1533 |
| 1467 if (FLAG_trace_inlining && | 1534 if (FLAG_trace_inlining && |
| 1468 (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized)) { | 1535 (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized)) { |
| 1469 OS::Print("Before Inlining of %s\n", flow_graph_-> | 1536 OS::Print("Before Inlining of %s\n", flow_graph_-> |
| 1470 parsed_function().function().ToFullyQualifiedCString()); | 1537 parsed_function().function().ToFullyQualifiedCString()); |
| 1471 FlowGraphPrinter printer(*flow_graph_); | 1538 FlowGraphPrinter printer(*flow_graph_); |
| 1472 printer.PrintBlocks(); | 1539 printer.PrintBlocks(); |
| 1473 } | 1540 } |
| 1474 | 1541 |
| 1475 CallSiteInliner inliner(flow_graph_); | 1542 CallSiteInliner inliner(flow_graph_); |
| 1476 inliner.InlineCalls(); | 1543 inliner.InlineCalls(); |
| 1544 if (FLAG_print_inline_tree) { |
| 1545 inliner.PrintInlinedInfo(top); |
| 1546 } |
| 1477 | 1547 |
| 1478 if (inliner.inlined()) { | 1548 if (inliner.inlined()) { |
| 1479 flow_graph_->DiscoverBlocks(); | 1549 flow_graph_->DiscoverBlocks(); |
| 1480 if (FLAG_trace_inlining) { | 1550 if (FLAG_trace_inlining) { |
| 1481 OS::Print("Inlining growth factor: %f\n", inliner.GrowthFactor()); | 1551 OS::Print("Inlining growth factor: %f\n", inliner.GrowthFactor()); |
| 1482 if (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized) { | 1552 if (FLAG_print_flow_graph || FLAG_print_flow_graph_optimized) { |
| 1483 OS::Print("After Inlining of %s\n", flow_graph_-> | 1553 OS::Print("After Inlining of %s\n", flow_graph_-> |
| 1484 parsed_function().function().ToFullyQualifiedCString()); | 1554 parsed_function().function().ToFullyQualifiedCString()); |
| 1485 FlowGraphPrinter printer(*flow_graph_); | 1555 FlowGraphPrinter printer(*flow_graph_); |
| 1486 printer.PrintBlocks(); | 1556 printer.PrintBlocks(); |
| 1487 } | 1557 } |
| 1488 } | 1558 } |
| 1489 } | 1559 } |
| 1490 } | 1560 } |
| 1491 | 1561 |
| 1492 } // namespace dart | 1562 } // namespace dart |
| OLD | NEW |