Index: runtime/vm/flow_graph_inliner.cc |
diff --git a/runtime/vm/flow_graph_inliner.cc b/runtime/vm/flow_graph_inliner.cc |
index 95e634b1a1815002f21702aa1889bdf8a54acbf0..8f8eeecc88ec002fa95f244265c36c327e2e5c79 100644 |
--- a/runtime/vm/flow_graph_inliner.cc |
+++ b/runtime/vm/flow_graph_inliner.cc |
@@ -474,7 +474,7 @@ class PolymorphicInliner : public ValueObject { |
bool CheckInlinedDuplicate(const Function& target); |
bool CheckNonInlinedDuplicate(const Function& target); |
- bool TryInliningPoly(const CidRangeTarget& target); |
+ bool TryInliningPoly(const TargetInfo& target); |
bool TryInlineRecognizedMethod(intptr_t receiver_cid, const Function& target); |
TargetEntryInstr* BuildDecisionGraph(); |
@@ -1446,8 +1446,8 @@ PolymorphicInliner::PolymorphicInliner(CallSiteInliner* owner, |
call_(call), |
num_variants_(call->NumberOfChecks()), |
variants_(call->targets_), |
- inlined_variants_(), |
- non_inlined_variants_(new (zone()) CallTargets()), |
+ inlined_variants_(zone()), |
+ non_inlined_variants_(new (zone()) CallTargets(zone())), |
inlined_entries_(num_variants_), |
exit_collector_(new (Z) InlineExitCollector(owner->caller_graph(), call)), |
caller_function_(caller_function), |
@@ -1480,7 +1480,7 @@ intptr_t PolymorphicInliner::AllocateBlockId() const { |
// * JoinEntry: the inlined body is shared and this is a subsequent variant. |
bool PolymorphicInliner::CheckInlinedDuplicate(const Function& target) { |
for (intptr_t i = 0; i < inlined_variants_.length(); ++i) { |
- if ((target.raw() == inlined_variants_[i].target->raw()) && |
+ if ((target.raw() == inlined_variants_.TargetAt(i)->target->raw()) && |
!MethodRecognizer::PolymorphicTarget(target)) { |
// The call target is shared with a previous inlined variant. Share |
// the graph. This requires a join block at the entry, and edge-split |
@@ -1532,7 +1532,7 @@ bool PolymorphicInliner::CheckInlinedDuplicate(const Function& target) { |
bool PolymorphicInliner::CheckNonInlinedDuplicate(const Function& target) { |
for (intptr_t i = 0; i < non_inlined_variants_->length(); ++i) { |
- if (target.raw() == non_inlined_variants_->At(i).target->raw()) { |
+ if (target.raw() == non_inlined_variants_->TargetAt(i)->target->raw()) { |
return true; |
} |
} |
@@ -1541,11 +1541,11 @@ bool PolymorphicInliner::CheckNonInlinedDuplicate(const Function& target) { |
} |
-bool PolymorphicInliner::TryInliningPoly(const CidRangeTarget& range) { |
+bool PolymorphicInliner::TryInliningPoly(const TargetInfo& target_info) { |
if ((!FLAG_precompiled_mode || |
owner_->inliner_->use_speculative_inlining()) && |
- range.cid_start == range.cid_end && |
- TryInlineRecognizedMethod(range.cid_start, *range.target)) { |
+ target_info.cid_start == target_info.cid_end && |
+ TryInlineRecognizedMethod(target_info.cid_start, *target_info.target)) { |
owner_->inlined_ = true; |
return true; |
} |
@@ -1556,7 +1556,7 @@ bool PolymorphicInliner::TryInliningPoly(const CidRangeTarget& range) { |
} |
InlinedCallData call_data(call_, &arguments, caller_function_, |
caller_inlining_id_); |
- Function& target = Function::ZoneHandle(zone(), range.target->raw()); |
+ Function& target = Function::ZoneHandle(zone(), target_info.target->raw()); |
if (!owner_->TryInlining(target, call_->instance_call()->argument_names(), |
&call_data)) { |
return false; |
@@ -1575,8 +1575,8 @@ bool PolymorphicInliner::TryInliningPoly(const CidRangeTarget& range) { |
RedefinitionInstr* redefinition = new (Z) RedefinitionInstr(actual->Copy(Z)); |
redefinition->set_ssa_temp_index( |
owner_->caller_graph()->alloc_ssa_temp_index()); |
- if (range.cid_start == range.cid_end) { |
- redefinition->UpdateType(CompileType::FromCid(range.cid_start)); |
+ if (target_info.cid_start == target_info.cid_end) { |
+ redefinition->UpdateType(CompileType::FromCid(target_info.cid_start)); |
} |
redefinition->InsertAfter(callee_graph->graph_entry()->normal_entry()); |
Definition* stub = (*call_data.parameter_stubs)[0]; |
@@ -1696,7 +1696,7 @@ TargetEntryInstr* PolymorphicInliner::BuildDecisionGraph() { |
cursor = AppendInstruction(cursor, load_cid); |
bool follow_with_deopt = false; |
for (intptr_t i = 0; i < inlined_variants_.length(); ++i) { |
- const CidRangeTarget& variant = inlined_variants_[i]; |
+ const CidRange& variant = inlined_variants_[i]; |
bool test_is_range = (variant.cid_start != variant.cid_end); |
bool is_last_test = (i == inlined_variants_.length() - 1); |
// 1. Guard the body with a class id check. We don't need any check if |
@@ -1926,14 +1926,16 @@ TargetEntryInstr* PolymorphicInliner::BuildDecisionGraph() { |
} |
-static void TracePolyInlining(const CidRangeTarget& crt, |
+static void TracePolyInlining(const CallTargets& targets, |
+ intptr_t idx, |
intptr_t total, |
const char* message) { |
- String& name = String::Handle(crt.target->QualifiedUserVisibleName()); |
- int percent = total == 0 ? 0 : (100 * crt.count) / total; |
+ String& name = |
+ String::Handle(targets.TargetAt(idx)->target->QualifiedUserVisibleName()); |
+ int percent = total == 0 ? 0 : (100 * targets.TargetAt(idx)->count) / total; |
THR_Print("%s cid %" Pd "-%" Pd ": %" Pd "/%" Pd " %d%% %s\n", |
- name.ToCString(), crt.cid_start, crt.cid_end, crt.count, total, |
- percent, message); |
+ name.ToCString(), targets[idx].cid_start, targets[idx].cid_end, |
+ targets.TargetAt(idx)->count, total, percent, message); |
} |
@@ -1947,18 +1949,20 @@ void PolymorphicInliner::Inline() { |
intptr_t total = call_->total_call_count(); |
for (intptr_t var_idx = 0; var_idx < variants_.length(); ++var_idx) { |
+ TargetInfo* info = variants_.TargetAt(var_idx); |
if (variants_.length() > FLAG_max_polymorphic_checks) { |
- non_inlined_variants_->Add(variants_[var_idx]); |
+ non_inlined_variants_->Add(info); |
continue; |
} |
+ const Function& target = *variants_.TargetAt(var_idx)->target; |
+ const intptr_t count = variants_.TargetAt(var_idx)->count; |
+ |
// We we almost inlined all the cases then try a little harder to inline |
// the last two, because it's a big win if we inline all of them (compiler |
// can see all side effects). |
const bool try_harder = (var_idx >= variants_.length() - 2) && |
non_inlined_variants_->length() == 0; |
- const Function& target = *variants_[var_idx].target; |
- const intptr_t count = variants_[var_idx].count; |
intptr_t size = target.optimized_instruction_count(); |
bool small = (size != 0 && size < FLAG_inlining_size_threshold); |
@@ -1968,16 +1972,16 @@ void PolymorphicInliner::Inline() { |
// version. |
if (!try_harder && count < (total >> 5)) { |
TRACE_INLINING( |
- TracePolyInlining(variants_[var_idx], total, "way too infrequent")); |
- non_inlined_variants_->Add(variants_[var_idx]); |
+ TracePolyInlining(variants_, var_idx, total, "way too infrequent")); |
+ non_inlined_variants_->Add(info); |
continue; |
} |
// First check if this is the same target as an earlier inlined variant. |
if (CheckInlinedDuplicate(target)) { |
- TRACE_INLINING(TracePolyInlining(variants_[var_idx], total, |
+ TRACE_INLINING(TracePolyInlining(variants_, var_idx, total, |
"duplicate already inlined")); |
- inlined_variants_.Add(variants_[var_idx]); |
+ inlined_variants_.Add(info); |
continue; |
} |
@@ -1986,8 +1990,8 @@ void PolymorphicInliner::Inline() { |
// consider inlining for 6% of the cases. |
if (!try_harder && count < (total >> (small ? 4 : 3))) { |
TRACE_INLINING( |
- TracePolyInlining(variants_[var_idx], total, "too infrequent")); |
- non_inlined_variants_->Add(variants_[var_idx]); |
+ TracePolyInlining(variants_, var_idx, total, "too infrequent")); |
+ non_inlined_variants_->Add(&variants_[var_idx]); |
continue; |
} |
@@ -1996,19 +2000,19 @@ void PolymorphicInliner::Inline() { |
// to inline this variant. |
if (CheckNonInlinedDuplicate(target)) { |
TRACE_INLINING( |
- TracePolyInlining(variants_[var_idx], total, "already not inlined")); |
- non_inlined_variants_->Add(variants_[var_idx]); |
+ TracePolyInlining(variants_, var_idx, total, "already not inlined")); |
+ non_inlined_variants_->Add(&variants_[var_idx]); |
continue; |
} |
// Make an inlining decision. |
- if (TryInliningPoly(variants_[var_idx])) { |
- TRACE_INLINING(TracePolyInlining(variants_[var_idx], total, "inlined")); |
- inlined_variants_.Add(variants_[var_idx]); |
+ if (TryInliningPoly(*info)) { |
+ TRACE_INLINING(TracePolyInlining(variants_, var_idx, total, "inlined")); |
+ inlined_variants_.Add(&variants_[var_idx]); |
} else { |
TRACE_INLINING( |
- TracePolyInlining(variants_[var_idx], total, "not inlined")); |
- non_inlined_variants_->Add(variants_[var_idx]); |
+ TracePolyInlining(variants_, var_idx, total, "not inlined")); |
+ non_inlined_variants_->Add(&variants_[var_idx]); |
} |
} |
@@ -2269,20 +2273,6 @@ static intptr_t PrepareInlineIndexedOp(FlowGraph* flow_graph, |
} |
-static Instruction* GetCheckClass(FlowGraph* flow_graph, |
- Definition* to_check, |
- const ICData& unary_checks, |
- intptr_t deopt_id, |
- TokenPosition token_pos) { |
- if ((unary_checks.NumberOfUsedChecks() == 1) && |
- unary_checks.HasReceiverClassId(kSmiCid)) { |
- return new (Z) CheckSmiInstr(new (Z) Value(to_check), deopt_id, token_pos); |
- } |
- return new (Z) CheckClassInstr(new (Z) Value(to_check), deopt_id, |
- unary_checks, token_pos); |
-} |
- |
- |
static bool InlineGetIndexed(FlowGraph* flow_graph, |
MethodRecognizer::Kind kind, |
Instruction* call, |
@@ -2333,7 +2323,7 @@ static bool InlineSetIndexed(FlowGraph* flow_graph, |
Instruction* call, |
Definition* receiver, |
TokenPosition token_pos, |
- const ICData& value_check, |
+ const Cids* value_check, |
TargetEntryInstr** entry, |
Definition** last) { |
intptr_t array_cid = MethodRecognizer::MethodKindToReceiverCid(kind); |
@@ -2428,12 +2418,12 @@ static bool InlineSetIndexed(FlowGraph* flow_graph, |
// No need to class check stores to Int32 and Uint32 arrays because |
// we insert unboxing instructions below which include a class check. |
if ((array_cid != kTypedDataUint32ArrayCid) && |
- (array_cid != kTypedDataInt32ArrayCid) && !value_check.IsNull()) { |
+ (array_cid != kTypedDataInt32ArrayCid) && value_check != NULL) { |
// No store barrier needed because checked value is a smi, an unboxed mint, |
// an unboxed double, an unboxed Float32x4, or unboxed Int32x4. |
needs_store_barrier = kNoStoreBarrier; |
- Instruction* check = GetCheckClass(flow_graph, stored_value, value_check, |
- call->deopt_id(), call->token_pos()); |
+ Instruction* check = flow_graph->CreateCheckClass( |
+ stored_value, *value_check, call->deopt_id(), call->token_pos()); |
cursor = |
flow_graph->AppendTo(cursor, check, call->env(), FlowGraph::kEffect); |
} |
@@ -2696,7 +2686,7 @@ static bool InlineByteArrayBaseStore(FlowGraph* flow_graph, |
i_call = call->AsInstanceCall(); |
} |
ASSERT(i_call != NULL); |
- ICData& value_check = ICData::ZoneHandle(Z); |
+ Cids* value_check = NULL; |
switch (view_cid) { |
case kTypedDataInt8ArrayCid: |
case kTypedDataUint8ArrayCid: |
@@ -2706,46 +2696,30 @@ static bool InlineByteArrayBaseStore(FlowGraph* flow_graph, |
case kTypedDataInt16ArrayCid: |
case kTypedDataUint16ArrayCid: { |
// Check that value is always smi. |
- value_check = ICData::New(flow_graph->function(), i_call->function_name(), |
- Object::empty_array(), // Dummy args. descr. |
- Thread::kNoDeoptId, 1, false); |
- value_check.AddReceiverCheck(kSmiCid, target); |
+ value_check = Cids::CreateMonomorphic(Z, kSmiCid); |
break; |
} |
case kTypedDataInt32ArrayCid: |
case kTypedDataUint32ArrayCid: |
// On 64-bit platforms assume that stored value is always a smi. |
if (kSmiBits >= 32) { |
- value_check = |
- ICData::New(flow_graph->function(), i_call->function_name(), |
- Object::empty_array(), // Dummy args. descr. |
- Thread::kNoDeoptId, 1, false); |
- value_check.AddReceiverCheck(kSmiCid, target); |
+ value_check = Cids::CreateMonomorphic(Z, kSmiCid); |
} |
break; |
case kTypedDataFloat32ArrayCid: |
case kTypedDataFloat64ArrayCid: { |
// Check that value is always double. |
- value_check = ICData::New(flow_graph->function(), i_call->function_name(), |
- Object::empty_array(), // Dummy args. descr. |
- Thread::kNoDeoptId, 1, false); |
- value_check.AddReceiverCheck(kDoubleCid, target); |
+ value_check = Cids::CreateMonomorphic(Z, kDoubleCid); |
break; |
} |
case kTypedDataInt32x4ArrayCid: { |
// Check that value is always Int32x4. |
- value_check = ICData::New(flow_graph->function(), i_call->function_name(), |
- Object::empty_array(), // Dummy args. descr. |
- Thread::kNoDeoptId, 1, false); |
- value_check.AddReceiverCheck(kInt32x4Cid, target); |
+ value_check = Cids::CreateMonomorphic(Z, kInt32x4Cid); |
break; |
} |
case kTypedDataFloat32x4ArrayCid: { |
// Check that value is always Float32x4. |
- value_check = ICData::New(flow_graph->function(), i_call->function_name(), |
- Object::empty_array(), // Dummy args. descr. |
- Thread::kNoDeoptId, 1, false); |
- value_check.AddReceiverCheck(kFloat32x4Cid, target); |
+ value_check = Cids::CreateMonomorphic(Z, kFloat32x4Cid); |
break; |
} |
default: |
@@ -2754,9 +2728,9 @@ static bool InlineByteArrayBaseStore(FlowGraph* flow_graph, |
} |
Definition* stored_value = call->ArgumentAt(2); |
- if (!value_check.IsNull()) { |
- Instruction* check = GetCheckClass(flow_graph, stored_value, value_check, |
- call->deopt_id(), call->token_pos()); |
+ if (value_check != NULL) { |
+ Instruction* check = flow_graph->CreateCheckClass( |
+ stored_value, *value_check, call->deopt_id(), call->token_pos()); |
cursor = |
flow_graph->AppendTo(cursor, check, call->env(), FlowGraph::kEffect); |
} |
@@ -2916,9 +2890,8 @@ bool FlowGraphInliner::TryReplaceInstanceCallWithInline( |
// Insert receiver class check if needed. |
if (MethodRecognizer::PolymorphicTarget(target) || |
flow_graph->InstanceCallNeedsClassCheck(call, target.kind())) { |
- Instruction* check = GetCheckClass( |
- flow_graph, call->ArgumentAt(0), |
- ICData::ZoneHandle(Z, call->ic_data()->AsUnaryClassChecks()), |
+ Instruction* check = flow_graph->CreateCheckClass( |
+ call->ArgumentAt(0), *Cids::Create(Z, *call->ic_data(), 0), |
call->deopt_id(), call->token_pos()); |
flow_graph->InsertBefore(call, check, call->env(), FlowGraph::kEffect); |
} |
@@ -3405,7 +3378,6 @@ bool FlowGraphInliner::TryInlineRecognizedMethod(FlowGraph* flow_graph, |
const ICData& ic_data, |
TargetEntryInstr** entry, |
Definition** last) { |
- ICData& value_check = ICData::ZoneHandle(Z); |
MethodRecognizer::Kind kind = MethodRecognizer::RecognizeKind(target); |
switch (kind) { |
// Recognized [] operators. |
@@ -3446,59 +3418,61 @@ bool FlowGraphInliner::TryInlineRecognizedMethod(FlowGraph* flow_graph, |
case MethodRecognizer::kObjectArraySetIndexed: |
case MethodRecognizer::kGrowableArraySetIndexed: |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
- token_pos, value_check, entry, last); |
+ token_pos, /* value_check = */ NULL, entry, last); |
case MethodRecognizer::kInt8ArraySetIndexed: |
case MethodRecognizer::kUint8ArraySetIndexed: |
case MethodRecognizer::kUint8ClampedArraySetIndexed: |
case MethodRecognizer::kExternalUint8ArraySetIndexed: |
case MethodRecognizer::kExternalUint8ClampedArraySetIndexed: |
case MethodRecognizer::kInt16ArraySetIndexed: |
- case MethodRecognizer::kUint16ArraySetIndexed: |
+ case MethodRecognizer::kUint16ArraySetIndexed: { |
// Optimistically assume Smi. |
if (ic_data.HasDeoptReason(ICData::kDeoptCheckSmi)) { |
// Optimistic assumption failed at least once. |
return false; |
} |
- value_check = ic_data.AsUnaryClassChecksForCid(kSmiCid, target); |
+ Cids* value_check = Cids::CreateMonomorphic(Z, kSmiCid); |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
token_pos, value_check, entry, last); |
+ } |
case MethodRecognizer::kInt32ArraySetIndexed: |
case MethodRecognizer::kUint32ArraySetIndexed: { |
// Value check not needed for Int32 and Uint32 arrays because they |
// implicitly contain unboxing instructions which check for right type. |
- ICData& value_check = ICData::Handle(); |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
- token_pos, value_check, entry, last); |
+ token_pos, /* value_check = */ NULL, entry, last); |
} |
case MethodRecognizer::kInt64ArraySetIndexed: |
if (!ShouldInlineInt64ArrayOps()) { |
return false; |
} |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
- token_pos, value_check, entry, last); |
+ token_pos, /* value_check = */ NULL, entry, last); |
case MethodRecognizer::kFloat32ArraySetIndexed: |
- case MethodRecognizer::kFloat64ArraySetIndexed: |
+ case MethodRecognizer::kFloat64ArraySetIndexed: { |
if (!CanUnboxDouble()) { |
return false; |
} |
- value_check = ic_data.AsUnaryClassChecksForCid(kDoubleCid, target); |
+ Cids* value_check = Cids::CreateMonomorphic(Z, kDoubleCid); |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
token_pos, value_check, entry, last); |
- case MethodRecognizer::kFloat32x4ArraySetIndexed: |
+ } |
+ case MethodRecognizer::kFloat32x4ArraySetIndexed: { |
if (!ShouldInlineSimd()) { |
return false; |
} |
- value_check = ic_data.AsUnaryClassChecksForCid(kFloat32x4Cid, target); |
- |
+ Cids* value_check = Cids::CreateMonomorphic(Z, kFloat32x4Cid); |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
token_pos, value_check, entry, last); |
- case MethodRecognizer::kFloat64x2ArraySetIndexed: |
+ } |
+ case MethodRecognizer::kFloat64x2ArraySetIndexed: { |
if (!ShouldInlineSimd()) { |
return false; |
} |
- value_check = ic_data.AsUnaryClassChecksForCid(kFloat64x2Cid, target); |
+ Cids* value_check = Cids::CreateMonomorphic(Z, kFloat64x2Cid); |
return InlineSetIndexed(flow_graph, kind, target, call, receiver, |
token_pos, value_check, entry, last); |
+ } |
case MethodRecognizer::kByteArrayBaseGetInt8: |
return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, |
kTypedDataInt8ArrayCid, entry, last); |