| Index: src/IceCfgNode.cpp
|
| diff --git a/src/IceCfgNode.cpp b/src/IceCfgNode.cpp
|
| index d57224b9f14a2949e5aa5f04061114054eab42eb..d683e7bd73cab2ac37fb9112f8a9c37c0e5212ca 100644
|
| --- a/src/IceCfgNode.cpp
|
| +++ b/src/IceCfgNode.cpp
|
| @@ -8,8 +8,8 @@
|
| //===----------------------------------------------------------------------===//
|
| ///
|
| /// \file
|
| -/// This file implements the CfgNode class, including the complexities
|
| -/// of instruction insertion and in-edge calculation.
|
| +/// This file implements the CfgNode class, including the complexities of
|
| +/// instruction insertion and in-edge calculation.
|
| ///
|
| //===----------------------------------------------------------------------===//
|
|
|
| @@ -29,17 +29,16 @@ namespace Ice {
|
| CfgNode::CfgNode(Cfg *Func, SizeT LabelNumber)
|
| : Func(Func), Number(LabelNumber) {}
|
|
|
| -// Returns the name the node was created with. If no name was given,
|
| -// it synthesizes a (hopefully) unique name.
|
| +// Returns the name the node was created with. If no name was given, it
|
| +// synthesizes a (hopefully) unique name.
|
| IceString CfgNode::getName() const {
|
| if (NameIndex >= 0)
|
| return Func->getIdentifierName(NameIndex);
|
| return "__" + std::to_string(getIndex());
|
| }
|
|
|
| -// Adds an instruction to either the Phi list or the regular
|
| -// instruction list. Validates that all Phis are added before all
|
| -// regular instructions.
|
| +// Adds an instruction to either the Phi list or the regular instruction list.
|
| +// Validates that all Phis are added before all regular instructions.
|
| void CfgNode::appendInst(Inst *Inst) {
|
| ++InstCountEstimate;
|
| if (InstPhi *Phi = llvm::dyn_cast<InstPhi>(Inst)) {
|
| @@ -53,11 +52,10 @@ void CfgNode::appendInst(Inst *Inst) {
|
| }
|
| }
|
|
|
| -// Renumbers the non-deleted instructions in the node. This needs to
|
| -// be done in preparation for live range analysis. The instruction
|
| -// numbers in a block must be monotonically increasing. The range of
|
| -// instruction numbers in a block, from lowest to highest, must not
|
| -// overlap with the range of any other block.
|
| +// Renumbers the non-deleted instructions in the node. This needs to be done in
|
| +// preparation for live range analysis. The instruction numbers in a block must
|
| +// be monotonically increasing. The range of instruction numbers in a block,
|
| +// from lowest to highest, must not overlap with the range of any other block.
|
| void CfgNode::renumberInstructions() {
|
| InstNumberT FirstNumber = Func->getNextInstNumber();
|
| for (Inst &I : Phis)
|
| @@ -67,10 +65,9 @@ void CfgNode::renumberInstructions() {
|
| InstCountEstimate = Func->getNextInstNumber() - FirstNumber;
|
| }
|
|
|
| -// When a node is created, the OutEdges are immediately known, but the
|
| -// InEdges have to be built up incrementally. After the CFG has been
|
| -// constructed, the computePredecessors() pass finalizes it by
|
| -// creating the InEdges list.
|
| +// When a node is created, the OutEdges are immediately known, but the InEdges
|
| +// have to be built up incrementally. After the CFG has been constructed, the
|
| +// computePredecessors() pass finalizes it by creating the InEdges list.
|
| void CfgNode::computePredecessors() {
|
| for (CfgNode *Succ : OutEdges)
|
| Succ->InEdges.push_back(this);
|
| @@ -80,17 +77,17 @@ void CfgNode::computeSuccessors() {
|
| OutEdges = Insts.rbegin()->getTerminatorEdges();
|
| }
|
|
|
| -// This does part 1 of Phi lowering, by creating a new dest variable
|
| -// for each Phi instruction, replacing the Phi instruction's dest with
|
| -// that variable, and adding an explicit assignment of the old dest to
|
| -// the new dest. For example,
|
| +// This does part 1 of Phi lowering, by creating a new dest variable for each
|
| +// Phi instruction, replacing the Phi instruction's dest with that variable,
|
| +// and adding an explicit assignment of the old dest to the new dest. For
|
| +// example,
|
| // a=phi(...)
|
| // changes to
|
| // "a_phi=phi(...); a=a_phi".
|
| //
|
| -// This is in preparation for part 2 which deletes the Phi
|
| -// instructions and appends assignment instructions to predecessor
|
| -// blocks. Note that this transformation preserves SSA form.
|
| +// This is in preparation for part 2 which deletes the Phi instructions and
|
| +// appends assignment instructions to predecessor blocks. Note that this
|
| +// transformation preserves SSA form.
|
| void CfgNode::placePhiLoads() {
|
| for (Inst &I : Phis) {
|
| auto Phi = llvm::dyn_cast<InstPhi>(&I);
|
| @@ -98,38 +95,35 @@ void CfgNode::placePhiLoads() {
|
| }
|
| }
|
|
|
| -// This does part 2 of Phi lowering. For each Phi instruction at each
|
| -// out-edge, create a corresponding assignment instruction, and add
|
| -// all the assignments near the end of this block. They need to be
|
| -// added before any branch instruction, and also if the block ends
|
| -// with a compare instruction followed by a branch instruction that we
|
| -// may want to fuse, it's better to insert the new assignments before
|
| -// the compare instruction. The tryOptimizedCmpxchgCmpBr() method
|
| -// assumes this ordering of instructions.
|
| +// This does part 2 of Phi lowering. For each Phi instruction at each out-edge,
|
| +// create a corresponding assignment instruction, and add all the assignments
|
| +// near the end of this block. They need to be added before any branch
|
| +// instruction, and also if the block ends with a compare instruction followed
|
| +// by a branch instruction that we may want to fuse, it's better to insert the
|
| +// new assignments before the compare instruction. The
|
| +// tryOptimizedCmpxchgCmpBr() method assumes this ordering of instructions.
|
| //
|
| -// Note that this transformation takes the Phi dest variables out of
|
| -// SSA form, as there may be assignments to the dest variable in
|
| -// multiple blocks.
|
| +// Note that this transformation takes the Phi dest variables out of SSA form,
|
| +// as there may be assignments to the dest variable in multiple blocks.
|
| void CfgNode::placePhiStores() {
|
| // Find the insertion point.
|
| InstList::iterator InsertionPoint = Insts.end();
|
| - // Every block must end in a terminator instruction, and therefore
|
| - // must have at least one instruction, so it's valid to decrement
|
| - // InsertionPoint (but assert just in case).
|
| + // Every block must end in a terminator instruction, and therefore must have
|
| + // at least one instruction, so it's valid to decrement InsertionPoint (but
|
| + // assert just in case).
|
| assert(InsertionPoint != Insts.begin());
|
| --InsertionPoint;
|
| - // Confirm that InsertionPoint is a terminator instruction. Calling
|
| - // getTerminatorEdges() on a non-terminator instruction will cause
|
| - // an llvm_unreachable().
|
| + // Confirm that InsertionPoint is a terminator instruction. Calling
|
| + // getTerminatorEdges() on a non-terminator instruction will cause an
|
| + // llvm_unreachable().
|
| (void)InsertionPoint->getTerminatorEdges();
|
| // SafeInsertionPoint is always immediately before the terminator
|
| - // instruction. If the block ends in a compare and conditional
|
| - // branch, it's better to place the Phi store before the compare so
|
| - // as not to interfere with compare/branch fusing. However, if the
|
| - // compare instruction's dest operand is the same as the new
|
| - // assignment statement's source operand, this can't be done due to
|
| - // data dependences, so we need to fall back to the
|
| - // SafeInsertionPoint. To illustrate:
|
| + // instruction. If the block ends in a compare and conditional branch, it's
|
| + // better to place the Phi store before the compare so as not to interfere
|
| + // with compare/branch fusing. However, if the compare instruction's dest
|
| + // operand is the same as the new assignment statement's source operand, this
|
| + // can't be done due to data dependences, so we need to fall back to the
|
| + // SafeInsertionPoint. To illustrate:
|
| // ; <label>:95
|
| // %97 = load i8* %96, align 1
|
| // %98 = icmp ne i8 %97, 0
|
| @@ -148,9 +142,8 @@ void CfgNode::placePhiStores() {
|
| // %100 = %100_phi
|
| // %101 = %101_phi
|
| //
|
| - // TODO(stichnot): It may be possible to bypass this whole
|
| - // SafeInsertionPoint mechanism. If a source basic block ends in a
|
| - // conditional branch:
|
| + // TODO(stichnot): It may be possible to bypass this whole SafeInsertionPoint
|
| + // mechanism. If a source basic block ends in a conditional branch:
|
| // labelSource:
|
| // ...
|
| // br i1 %foo, label %labelTrue, label %labelFalse
|
| @@ -160,17 +153,17 @@ void CfgNode::placePhiStores() {
|
| // then we actually know the constant i1 value of the Phi operand:
|
| // labelTrue:
|
| // %bar = phi i1 [ true, %labelSource ], ...
|
| - // It seems that this optimization should be done by clang or opt,
|
| - // but we could also do it here.
|
| + // It seems that this optimization should be done by clang or opt, but we
|
| + // could also do it here.
|
| InstList::iterator SafeInsertionPoint = InsertionPoint;
|
| - // Keep track of the dest variable of a compare instruction, so that
|
| - // we insert the new instruction at the SafeInsertionPoint if the
|
| - // compare's dest matches the Phi-lowered assignment's source.
|
| + // Keep track of the dest variable of a compare instruction, so that we
|
| + // insert the new instruction at the SafeInsertionPoint if the compare's dest
|
| + // matches the Phi-lowered assignment's source.
|
| Variable *CmpInstDest = nullptr;
|
| - // If the current insertion point is at a conditional branch
|
| - // instruction, and the previous instruction is a compare
|
| - // instruction, then we move the insertion point before the compare
|
| - // instruction so as not to interfere with compare/branch fusing.
|
| + // If the current insertion point is at a conditional branch instruction, and
|
| + // the previous instruction is a compare instruction, then we move the
|
| + // insertion point before the compare instruction so as not to interfere with
|
| + // compare/branch fusing.
|
| if (InstBr *Branch = llvm::dyn_cast<InstBr>(InsertionPoint)) {
|
| if (!Branch->isUnconditional()) {
|
| if (InsertionPoint != Insts.begin()) {
|
| @@ -209,13 +202,12 @@ void CfgNode::deletePhis() {
|
| I.setDeleted();
|
| }
|
|
|
| -// Splits the edge from Pred to this node by creating a new node and
|
| -// hooking up the in and out edges appropriately. (The EdgeIndex
|
| -// parameter is only used to make the new node's name unique when
|
| -// there are multiple edges between the same pair of nodes.) The new
|
| -// node's instruction list is initialized to the empty list, with no
|
| -// terminator instruction. There must not be multiple edges from Pred
|
| -// to this node so all Inst::getTerminatorEdges implementations must
|
| +// Splits the edge from Pred to this node by creating a new node and hooking up
|
| +// the in and out edges appropriately. (The EdgeIndex parameter is only used to
|
| +// make the new node's name unique when there are multiple edges between the
|
| +// same pair of nodes.) The new node's instruction list is initialized to the
|
| +// empty list, with no terminator instruction. There must not be multiple edges
|
| +// from Pred to this node so all Inst::getTerminatorEdges implementations must
|
| // not contain duplicates.
|
| CfgNode *CfgNode::splitIncomingEdge(CfgNode *Pred, SizeT EdgeIndex) {
|
| CfgNode *NewNode = Func->makeNode();
|
| @@ -227,8 +219,8 @@ CfgNode *CfgNode::splitIncomingEdge(CfgNode *Pred, SizeT EdgeIndex) {
|
| if (BuildDefs::dump())
|
| NewNode->setName("split_" + Pred->getName() + "_" + getName() + "_" +
|
| std::to_string(EdgeIndex));
|
| - // The new node is added to the end of the node list, and will later
|
| - // need to be sorted into a reasonable topological order.
|
| + // The new node is added to the end of the node list, and will later need to
|
| + // be sorted into a reasonable topological order.
|
| NewNode->setNeedsPlacement(true);
|
| // Repoint Pred's out-edge.
|
| bool Found = false;
|
| @@ -279,31 +271,31 @@ bool sameVarOrReg(const Variable *Var, const Operand *Opnd) {
|
|
|
| } // end of anonymous namespace
|
|
|
| -// This the "advanced" version of Phi lowering for a basic block, in contrast to
|
| -// the simple version that lowers through assignments involving temporaries.
|
| +// This the "advanced" version of Phi lowering for a basic block, in contrast
|
| +// to the simple version that lowers through assignments involving temporaries.
|
| //
|
| // All Phi instructions in a basic block are conceptually executed in parallel.
|
| // However, if we lower Phis early and commit to a sequential ordering, we may
|
| // end up creating unnecessary interferences which lead to worse register
|
| -// allocation. Delaying Phi scheduling until after register allocation can help
|
| -// unless there are no free registers for shuffling registers or stack slots and
|
| -// spilling becomes necessary.
|
| +// allocation. Delaying Phi scheduling until after register allocation can help
|
| +// unless there are no free registers for shuffling registers or stack slots
|
| +// and spilling becomes necessary.
|
| //
|
| // The advanced Phi lowering starts by finding a topological sort of the Phi
|
| -// instructions, where "A=B" comes before "B=C" due to the anti-dependence on B.
|
| -// Preexisting register assignments are considered in the topological sort. If
|
| -// a topological sort is not possible due to a cycle, the cycle is broken by
|
| -// introducing a non-parallel temporary. For example, a cycle arising from a
|
| -// permutation like "A=B;B=C;C=A" can become "T=A;A=B;B=C;C=T". All else being
|
| +// instructions, where "A=B" comes before "B=C" due to the anti-dependence on
|
| +// B. Preexisting register assignments are considered in the topological sort.
|
| +// If a topological sort is not possible due to a cycle, the cycle is broken by
|
| +// introducing a non-parallel temporary. For example, a cycle arising from a
|
| +// permutation like "A=B;B=C;C=A" can become "T=A;A=B;B=C;C=T". All else being
|
| // equal, prefer to schedule assignments with register-allocated Src operands
|
| // earlier, in case that register becomes free afterwards, and prefer to
|
| // schedule assignments with register-allocated Dest variables later, to keep
|
| // that register free for longer.
|
| //
|
| // Once the ordering is determined, the Cfg edge is split and the assignment
|
| -// list is lowered by the target lowering layer. Since the assignment lowering
|
| +// list is lowered by the target lowering layer. Since the assignment lowering
|
| // may create new infinite-weight temporaries, a follow-on register allocation
|
| -// pass will be needed. To prepare for this, liveness (including live range
|
| +// pass will be needed. To prepare for this, liveness (including live range
|
| // calculation) of the split nodes needs to be calculated, and liveness of the
|
| // original node need to be updated to "undo" the effects of the phi
|
| // assignments.
|
| @@ -315,7 +307,7 @@ bool sameVarOrReg(const Variable *Var, const Operand *Opnd) {
|
| // allocation pass is run, focusing only on pre-colored and infinite-weight
|
| // variables, similar to Om1 register allocation (except without the need to
|
| // specially compute these variables' live ranges, since they have already been
|
| -// precisely calculated). The register allocator in this mode needs the ability
|
| +// precisely calculated). The register allocator in this mode needs the ability
|
| // to forcibly spill and reload registers in case none are naturally available.
|
| void CfgNode::advancedPhiLowering() {
|
| if (getPhis().empty())
|
| @@ -363,17 +355,16 @@ void CfgNode::advancedPhiLowering() {
|
| Desc[I].Src = Src;
|
| Desc[I].Processed = false;
|
| Desc[I].NumPred = 0;
|
| - // Cherry-pick any trivial assignments, so that they don't
|
| - // contribute to the running complexity of the topological sort.
|
| + // Cherry-pick any trivial assignments, so that they don't contribute to
|
| + // the running complexity of the topological sort.
|
| if (sameVarOrReg(Dest, Src)) {
|
| Desc[I].Processed = true;
|
| --Remaining;
|
| if (Dest != Src)
|
| - // If Dest and Src are syntactically the same, don't bother
|
| - // adding the assignment, because in all respects it would
|
| - // be redundant, and if Dest/Src are on the stack, the
|
| - // target lowering may naively decide to lower it using a
|
| - // temporary register.
|
| + // If Dest and Src are syntactically the same, don't bother adding
|
| + // the assignment, because in all respects it would be redundant, and
|
| + // if Dest/Src are on the stack, the target lowering may naively
|
| + // decide to lower it using a temporary register.
|
| Split->appendInst(InstAssign::create(Func, Dest, Src));
|
| }
|
| }
|
| @@ -387,8 +378,8 @@ void CfgNode::advancedPhiLowering() {
|
| if (Desc[J].Processed)
|
| continue;
|
| if (I != J) {
|
| - // There shouldn't be two Phis with the same Dest variable
|
| - // or register.
|
| + // There shouldn't be two Phis with the same Dest variable or
|
| + // register.
|
| assert(!sameVarOrReg(Dest, Desc[J].Dest));
|
| }
|
| const Operand *Src = Desc[J].Src;
|
| @@ -403,8 +394,7 @@ void CfgNode::advancedPhiLowering() {
|
| constexpr int32_t WeightNoPreds = 4;
|
| // Prefer Src as a register because the register might free up.
|
| constexpr int32_t WeightSrcIsReg = 2;
|
| - // Prefer Dest not as a register because the register stays free
|
| - // longer.
|
| + // Prefer Dest not as a register because the register stays free longer.
|
| constexpr int32_t WeightDestNotReg = 1;
|
|
|
| for (size_t I = 0; I < NumPhis; ++I) {
|
| @@ -421,11 +411,10 @@ void CfgNode::advancedPhiLowering() {
|
| Desc[I].Weight = Weight;
|
| }
|
|
|
| - // Repeatedly choose and process the best candidate in the
|
| - // topological sort, until no candidates remain. This
|
| - // implementation is O(N^2) where N is the number of Phi
|
| - // instructions, but with a small constant factor compared to a
|
| - // likely implementation of O(N) topological sort.
|
| + // Repeatedly choose and process the best candidate in the topological
|
| + // sort, until no candidates remain. This implementation is O(N^2) where N
|
| + // is the number of Phi instructions, but with a small constant factor
|
| + // compared to a likely implementation of O(N) topological sort.
|
| for (; Remaining; --Remaining) {
|
| size_t BestIndex = 0;
|
| int32_t BestWeight = -1;
|
| @@ -448,9 +437,9 @@ void CfgNode::advancedPhiLowering() {
|
| // Break a cycle by introducing a temporary.
|
| if (Desc[BestIndex].NumPred) {
|
| bool Found = false;
|
| - // If the target instruction "A=B" is part of a cycle, find
|
| - // the "X=A" assignment in the cycle because it will have to
|
| - // be rewritten as "X=tmp".
|
| + // If the target instruction "A=B" is part of a cycle, find the "X=A"
|
| + // assignment in the cycle because it will have to be rewritten as
|
| + // "X=tmp".
|
| for (size_t J = 0; !Found && J < NumPhis; ++J) {
|
| if (Desc[J].Processed)
|
| continue;
|
| @@ -470,9 +459,8 @@ void CfgNode::advancedPhiLowering() {
|
| // Now that a cycle (if any) has been broken, create the actual
|
| // assignment.
|
| Split->appendInst(InstAssign::create(Func, Dest, Src));
|
| - // Update NumPred for all Phi assignments using this Phi's Src
|
| - // as their Dest variable. Also update Weight if NumPred
|
| - // dropped from 1 to 0.
|
| + // Update NumPred for all Phi assignments using this Phi's Src as their
|
| + // Dest variable. Also update Weight if NumPred dropped from 1 to 0.
|
| if (auto Var = llvm::dyn_cast<Variable>(Src)) {
|
| for (size_t I = 0; I < NumPhis; ++I) {
|
| if (Desc[I].Processed)
|
| @@ -492,10 +480,9 @@ void CfgNode::advancedPhiLowering() {
|
| }
|
| }
|
|
|
| -// Does address mode optimization. Pass each instruction to the
|
| -// TargetLowering object. If it returns a new instruction
|
| -// (representing the optimized address mode), then insert the new
|
| -// instruction and delete the old.
|
| +// Does address mode optimization. Pass each instruction to the TargetLowering
|
| +// object. If it returns a new instruction (representing the optimized address
|
| +// mode), then insert the new instruction and delete the old.
|
| void CfgNode::doAddressOpt() {
|
| TargetLowering *Target = Func->getTarget();
|
| LoweringContext &Context = Target->getContext();
|
| @@ -527,8 +514,8 @@ void CfgNode::doNopInsertion(RandomNumberGenerator &RNG) {
|
| }
|
| }
|
|
|
| -// Drives the target lowering. Passes the current instruction and the
|
| -// next non-deleted instruction for target lowering.
|
| +// Drives the target lowering. Passes the current instruction and the next
|
| +// non-deleted instruction for target lowering.
|
| void CfgNode::genCode() {
|
| TargetLowering *Target = Func->getTarget();
|
| LoweringContext &Context = Target->getContext();
|
| @@ -563,24 +550,23 @@ void CfgNode::livenessLightweight() {
|
| }
|
| }
|
|
|
| -// Performs liveness analysis on the block. Returns true if the
|
| -// incoming liveness changed from before, false if it stayed the same.
|
| -// (If it changes, the node's predecessors need to be processed
|
| -// again.)
|
| +// Performs liveness analysis on the block. Returns true if the incoming
|
| +// liveness changed from before, false if it stayed the same. (If it changes,
|
| +// the node's predecessors need to be processed again.)
|
| bool CfgNode::liveness(Liveness *Liveness) {
|
| SizeT NumVars = Liveness->getNumVarsInNode(this);
|
| LivenessBV Live(NumVars);
|
| LiveBeginEndMap *LiveBegin = nullptr;
|
| LiveBeginEndMap *LiveEnd = nullptr;
|
| - // Mark the beginning and ending of each variable's live range
|
| - // with the sentinel instruction number 0.
|
| + // Mark the beginning and ending of each variable's live range with the
|
| + // sentinel instruction number 0.
|
| if (Liveness->getMode() == Liveness_Intervals) {
|
| LiveBegin = Liveness->getLiveBegin(this);
|
| LiveEnd = Liveness->getLiveEnd(this);
|
| LiveBegin->clear();
|
| LiveEnd->clear();
|
| - // Guess that the number of live ranges beginning is roughly the
|
| - // number of instructions, and same for live ranges ending.
|
| + // Guess that the number of live ranges beginning is roughly the number of
|
| + // instructions, and same for live ranges ending.
|
| LiveBegin->reserve(getInstCountEstimate());
|
| LiveEnd->reserve(getInstCountEstimate());
|
| }
|
| @@ -603,9 +589,8 @@ bool CfgNode::liveness(Liveness *Liveness) {
|
| continue;
|
| I.liveness(I.getNumber(), Live, Liveness, LiveBegin, LiveEnd);
|
| }
|
| - // Process phis in forward order so that we can override the
|
| - // instruction number to be that of the earliest phi instruction in
|
| - // the block.
|
| + // Process phis in forward order so that we can override the instruction
|
| + // number to be that of the earliest phi instruction in the block.
|
| SizeT NumNonDeadPhis = 0;
|
| InstNumberT FirstPhiNumber = Inst::NumberSentinel;
|
| for (Inst &I : Phis) {
|
| @@ -617,18 +602,17 @@ bool CfgNode::liveness(Liveness *Liveness) {
|
| ++NumNonDeadPhis;
|
| }
|
|
|
| - // When using the sparse representation, after traversing the
|
| - // instructions in the block, the Live bitvector should only contain
|
| - // set bits for global variables upon block entry. We validate this
|
| - // by shrinking the Live vector and then testing it against the
|
| - // pre-shrunk version. (The shrinking is required, but the
|
| - // validation is not.)
|
| + // When using the sparse representation, after traversing the instructions in
|
| + // the block, the Live bitvector should only contain set bits for global
|
| + // variables upon block entry. We validate this by shrinking the Live vector
|
| + // and then testing it against the pre-shrunk version. (The shrinking is
|
| + // required, but the validation is not.)
|
| LivenessBV LiveOrig = Live;
|
| Live.resize(Liveness->getNumGlobalVars());
|
| if (Live != LiveOrig) {
|
| if (BuildDefs::dump()) {
|
| - // This is a fatal liveness consistency error. Print some
|
| - // diagnostics and abort.
|
| + // This is a fatal liveness consistency error. Print some diagnostics and
|
| + // abort.
|
| Ostream &Str = Func->getContext()->getStrDump();
|
| Func->resetCurrentNode();
|
| Str << "LiveOrig-Live =";
|
| @@ -657,13 +641,12 @@ bool CfgNode::liveness(Liveness *Liveness) {
|
| return Changed;
|
| }
|
|
|
| -// Once basic liveness is complete, compute actual live ranges. It is
|
| -// assumed that within a single basic block, a live range begins at
|
| -// most once and ends at most once. This is certainly true for pure
|
| -// SSA form. It is also true once phis are lowered, since each
|
| -// assignment to the phi-based temporary is in a different basic
|
| -// block, and there is a single read that ends the live in the basic
|
| -// block that contained the actual phi instruction.
|
| +// Once basic liveness is complete, compute actual live ranges. It is assumed
|
| +// that within a single basic block, a live range begins at most once and ends
|
| +// at most once. This is certainly true for pure SSA form. It is also true once
|
| +// phis are lowered, since each assignment to the phi-based temporary is in a
|
| +// different basic block, and there is a single read that ends the live in the
|
| +// basic block that contained the actual phi instruction.
|
| void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
|
| InstNumberT LastInstNum) {
|
| TimerMarker T1(TimerStack::TT_liveRange, Func);
|
| @@ -696,14 +679,13 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
|
| SizeT i1 = IBB == IBE ? NumVars : IBB->first;
|
| SizeT i2 = IEB == IEE ? NumVars : IEB->first;
|
| SizeT i = std::min(i1, i2);
|
| - // i1 is the Variable number of the next MapBegin entry, and i2 is
|
| - // the Variable number of the next MapEnd entry. If i1==i2, then
|
| - // the Variable's live range begins and ends in this block. If
|
| - // i1<i2, then i1's live range begins at instruction IBB->second
|
| - // and extends through the end of the block. If i1>i2, then i2's
|
| - // live range begins at the first instruction of the block and
|
| - // ends at IEB->second. In any case, we choose the lesser of i1
|
| - // and i2 and proceed accordingly.
|
| + // i1 is the Variable number of the next MapBegin entry, and i2 is the
|
| + // Variable number of the next MapEnd entry. If i1==i2, then the Variable's
|
| + // live range begins and ends in this block. If i1<i2, then i1's live range
|
| + // begins at instruction IBB->second and extends through the end of the
|
| + // block. If i1>i2, then i2's live range begins at the first instruction of
|
| + // the block and ends at IEB->second. In any case, we choose the lesser of
|
| + // i1 and i2 and proceed accordingly.
|
| InstNumberT LB = i == i1 ? IBB->second : FirstInstNum;
|
| InstNumberT LE = i == i2 ? IEB->second : LastInstNum + 1;
|
|
|
| @@ -711,9 +693,9 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
|
| if (LB > LE) {
|
| Var->addLiveRange(FirstInstNum, LE);
|
| Var->addLiveRange(LB, LastInstNum + 1);
|
| - // Assert that Var is a global variable by checking that its
|
| - // liveness index is less than the number of globals. This
|
| - // ensures that the LiveInAndOut[] access is valid.
|
| + // Assert that Var is a global variable by checking that its liveness
|
| + // index is less than the number of globals. This ensures that the
|
| + // LiveInAndOut[] access is valid.
|
| assert(i < Liveness->getNumGlobalVars());
|
| LiveInAndOut[i] = false;
|
| } else {
|
| @@ -734,8 +716,8 @@ void CfgNode::livenessAddIntervals(Liveness *Liveness, InstNumberT FirstInstNum,
|
| }
|
|
|
| // If this node contains only deleted instructions, and ends in an
|
| -// unconditional branch, contract the node by repointing all its
|
| -// in-edges to its successor.
|
| +// unconditional branch, contract the node by repointing all its in-edges to
|
| +// its successor.
|
| void CfgNode::contractIfEmpty() {
|
| if (InEdges.empty())
|
| return;
|
| @@ -751,10 +733,10 @@ void CfgNode::contractIfEmpty() {
|
| Branch->setDeleted();
|
| assert(OutEdges.size() == 1);
|
| CfgNode *Successor = OutEdges.front();
|
| - // Repoint all this node's in-edges to this node's successor, unless
|
| - // this node's successor is actually itself (in which case the
|
| - // statement "OutEdges.front()->InEdges.push_back(Pred)" could
|
| - // invalidate the iterator over this->InEdges).
|
| + // Repoint all this node's in-edges to this node's successor, unless this
|
| + // node's successor is actually itself (in which case the statement
|
| + // "OutEdges.front()->InEdges.push_back(Pred)" could invalidate the iterator
|
| + // over this->InEdges).
|
| if (Successor != this) {
|
| for (CfgNode *Pred : InEdges) {
|
| for (CfgNode *&I : Pred->OutEdges) {
|
| @@ -770,8 +752,8 @@ void CfgNode::contractIfEmpty() {
|
| }
|
|
|
| // Remove the in-edge to the successor to allow node reordering to make
|
| - // better decisions. For example it's more helpful to place a node after
|
| - // a reachable predecessor than an unreachable one (like the one we just
|
| + // better decisions. For example it's more helpful to place a node after a
|
| + // reachable predecessor than an unreachable one (like the one we just
|
| // contracted).
|
| Successor->InEdges.erase(
|
| std::find(Successor->InEdges.begin(), Successor->InEdges.end(), this));
|
| @@ -782,10 +764,10 @@ void CfgNode::contractIfEmpty() {
|
| void CfgNode::doBranchOpt(const CfgNode *NextNode) {
|
| TargetLowering *Target = Func->getTarget();
|
| // Find the first opportunity for branch optimization (which will be the last
|
| - // instruction in the block) and stop. This is sufficient unless there is some
|
| - // target lowering where we have the possibility of multiple optimizations per
|
| - // block. Take care with switch lowering as there are multiple unconditional
|
| - // branches and only the last can be deleted.
|
| + // instruction in the block) and stop. This is sufficient unless there is
|
| + // some target lowering where we have the possibility of multiple
|
| + // optimizations per block. Take care with switch lowering as there are
|
| + // multiple unconditional branches and only the last can be deleted.
|
| for (Inst &I : reverse_range(Insts)) {
|
| if (!I.isDeleted()) {
|
| Target->doBranchOpt(&I, NextNode);
|
| @@ -825,8 +807,8 @@ void emitRegisterUsage(Ostream &Str, const Cfg *Func, const CfgNode *Node,
|
| }
|
| }
|
| }
|
| - // Sort the variables by regnum so they are always printed in a
|
| - // familiar order.
|
| + // Sort the variables by regnum so they are always printed in a familiar
|
| + // order.
|
| std::sort(LiveRegs.begin(), LiveRegs.end(),
|
| [](const Variable *V1, const Variable *V2) {
|
| return V1->getRegNum() < V2->getRegNum();
|
| @@ -848,11 +830,11 @@ void emitLiveRangesEnded(Ostream &Str, const Cfg *Func, const Inst *Instr,
|
| return;
|
| bool First = true;
|
| Variable *Dest = Instr->getDest();
|
| - // Normally we increment the live count for the dest register. But
|
| - // we shouldn't if the instruction's IsDestNonKillable flag is set,
|
| - // because this means that the target lowering created this
|
| - // instruction as a non-SSA assignment; i.e., a different, previous
|
| - // instruction started the dest variable's live range.
|
| + // Normally we increment the live count for the dest register. But we
|
| + // shouldn't if the instruction's IsDestNonKillable flag is set, because this
|
| + // means that the target lowering created this instruction as a non-SSA
|
| + // assignment; i.e., a different, previous instruction started the dest
|
| + // variable's live range.
|
| if (!Instr->isDestNonKillable() && Dest && Dest->hasReg())
|
| ++LiveRegCount[Dest->getRegNum()];
|
| FOREACH_VAR_IN_INST(Var, *Instr) {
|
| @@ -877,8 +859,8 @@ void emitLiveRangesEnded(Ostream &Str, const Cfg *Func, const Inst *Instr,
|
| void updateStats(Cfg *Func, const Inst *I) {
|
| if (!BuildDefs::dump())
|
| return;
|
| - // Update emitted instruction count, plus fill/spill count for
|
| - // Variable operands without a physical register.
|
| + // Update emitted instruction count, plus fill/spill count for Variable
|
| + // operands without a physical register.
|
| if (uint32_t Count = I->getEmitInstCount()) {
|
| Func->getContext()->statsUpdateEmitted(Count);
|
| if (Variable *Dest = I->getDest()) {
|
| @@ -905,10 +887,10 @@ void CfgNode::emit(Cfg *Func) const {
|
| bool DecorateAsm =
|
| Liveness && Func->getContext()->getFlags().getDecorateAsm();
|
| Str << getAsmName() << ":\n";
|
| - // LiveRegCount keeps track of the number of currently live
|
| - // variables that each register is assigned to. Normally that would
|
| - // be only 0 or 1, but the register allocator's AllowOverlap
|
| - // inference allows it to be greater than 1 for short periods.
|
| + // LiveRegCount keeps track of the number of currently live variables that
|
| + // each register is assigned to. Normally that would be only 0 or 1, but the
|
| + // register allocator's AllowOverlap inference allows it to be greater than 1
|
| + // for short periods.
|
| std::vector<SizeT> LiveRegCount(Func->getTarget()->getNumRegisters());
|
| if (DecorateAsm) {
|
| constexpr bool IsLiveIn = true;
|
| @@ -925,15 +907,14 @@ void CfgNode::emit(Cfg *Func) const {
|
| if (I.isDeleted())
|
| continue;
|
| if (I.isRedundantAssign()) {
|
| - // Usually, redundant assignments end the live range of the src
|
| - // variable and begin the live range of the dest variable, with
|
| - // no net effect on the liveness of their register. However, if
|
| - // the register allocator infers the AllowOverlap condition,
|
| - // then this may be a redundant assignment that does not end the
|
| - // src variable's live range, in which case the active variable
|
| - // count for that register needs to be bumped. That normally
|
| - // would have happened as part of emitLiveRangesEnded(), but
|
| - // that isn't called for redundant assignments.
|
| + // Usually, redundant assignments end the live range of the src variable
|
| + // and begin the live range of the dest variable, with no net effect on
|
| + // the liveness of their register. However, if the register allocator
|
| + // infers the AllowOverlap condition, then this may be a redundant
|
| + // assignment that does not end the src variable's live range, in which
|
| + // case the active variable count for that register needs to be bumped.
|
| + // That normally would have happened as part of emitLiveRangesEnded(),
|
| + // but that isn't called for redundant assignments.
|
| Variable *Dest = I.getDest();
|
| if (DecorateAsm && Dest->hasReg() && !I.isLastUse(I.getSrc(0)))
|
| ++LiveRegCount[Dest->getRegNum()];
|
| @@ -966,41 +947,38 @@ public:
|
| BundleMaskLo(BundleSize - 1), BundleMaskHi(~BundleMaskLo) {}
|
| // Check whether we're currently within a bundle_lock region.
|
| bool isInBundleLockRegion() const { return BundleLockStart != End; }
|
| - // Check whether the current bundle_lock region has the align_to_end
|
| - // option.
|
| + // Check whether the current bundle_lock region has the align_to_end option.
|
| bool isAlignToEnd() const {
|
| assert(isInBundleLockRegion());
|
| return llvm::cast<InstBundleLock>(getBundleLockStart())->getOption() ==
|
| InstBundleLock::Opt_AlignToEnd;
|
| }
|
| - // Check whether the entire bundle_lock region falls within the same
|
| - // bundle.
|
| + // Check whether the entire bundle_lock region falls within the same bundle.
|
| bool isSameBundle() const {
|
| assert(isInBundleLockRegion());
|
| return SizeSnapshotPre == SizeSnapshotPost ||
|
| (SizeSnapshotPre & BundleMaskHi) ==
|
| ((SizeSnapshotPost - 1) & BundleMaskHi);
|
| }
|
| - // Get the bundle alignment of the first instruction of the
|
| - // bundle_lock region.
|
| + // Get the bundle alignment of the first instruction of the bundle_lock
|
| + // region.
|
| intptr_t getPreAlignment() const {
|
| assert(isInBundleLockRegion());
|
| return SizeSnapshotPre & BundleMaskLo;
|
| }
|
| - // Get the bundle alignment of the first instruction past the
|
| - // bundle_lock region.
|
| + // Get the bundle alignment of the first instruction past the bundle_lock
|
| + // region.
|
| intptr_t getPostAlignment() const {
|
| assert(isInBundleLockRegion());
|
| return SizeSnapshotPost & BundleMaskLo;
|
| }
|
| - // Get the iterator pointing to the bundle_lock instruction, e.g. to
|
| - // roll back the instruction iteration to that point.
|
| + // Get the iterator pointing to the bundle_lock instruction, e.g. to roll
|
| + // back the instruction iteration to that point.
|
| InstList::const_iterator getBundleLockStart() const {
|
| assert(isInBundleLockRegion());
|
| return BundleLockStart;
|
| }
|
| - // Set up bookkeeping when the bundle_lock instruction is first
|
| - // processed.
|
| + // Set up bookkeeping when the bundle_lock instruction is first processed.
|
| void enterBundleLock(InstList::const_iterator I) {
|
| assert(!isInBundleLockRegion());
|
| BundleLockStart = I;
|
| @@ -1009,18 +987,16 @@ public:
|
| Target->snapshotEmitState();
|
| assert(isInBundleLockRegion());
|
| }
|
| - // Update bookkeeping when the bundle_unlock instruction is
|
| - // processed.
|
| + // Update bookkeeping when the bundle_unlock instruction is processed.
|
| void enterBundleUnlock() {
|
| assert(isInBundleLockRegion());
|
| SizeSnapshotPost = Asm->getBufferSize();
|
| }
|
| - // Update bookkeeping when we are completely finished with the
|
| - // bundle_lock region.
|
| + // Update bookkeeping when we are completely finished with the bundle_lock
|
| + // region.
|
| void leaveBundleLockRegion() { BundleLockStart = End; }
|
| - // Check whether the instruction sequence fits within the current
|
| - // bundle, and if not, add nop padding to the end of the current
|
| - // bundle.
|
| + // Check whether the instruction sequence fits within the current bundle, and
|
| + // if not, add nop padding to the end of the current bundle.
|
| void padToNextBundle() {
|
| assert(isInBundleLockRegion());
|
| if (!isSameBundle()) {
|
| @@ -1032,8 +1008,8 @@ public:
|
| assert(Asm->getBufferSize() == SizeSnapshotPre);
|
| }
|
| }
|
| - // If align_to_end is specified, add padding such that the
|
| - // instruction sequences ends precisely at a bundle boundary.
|
| + // If align_to_end is specified, add padding such that the instruction
|
| + // sequences ends precisely at a bundle boundary.
|
| void padForAlignToEnd() {
|
| assert(isInBundleLockRegion());
|
| if (isAlignToEnd()) {
|
| @@ -1054,8 +1030,8 @@ public:
|
| private:
|
| Assembler *const Asm;
|
| TargetLowering *const Target;
|
| - // End is a sentinel value such that BundleLockStart==End implies
|
| - // that we are not in a bundle_lock region.
|
| + // End is a sentinel value such that BundleLockStart==End implies that we are
|
| + // not in a bundle_lock region.
|
| const InstList::const_iterator End;
|
| InstList::const_iterator BundleLockStart;
|
| const intptr_t BundleSize;
|
| @@ -1072,9 +1048,9 @@ private:
|
| void CfgNode::emitIAS(Cfg *Func) const {
|
| Func->setCurrentNode(this);
|
| Assembler *Asm = Func->getAssembler<>();
|
| - // TODO(stichnot): When sandboxing, defer binding the node label
|
| - // until just before the first instruction is emitted, to reduce the
|
| - // chance that a padding nop is a branch target.
|
| + // TODO(stichnot): When sandboxing, defer binding the node label until just
|
| + // before the first instruction is emitted, to reduce the chance that a
|
| + // padding nop is a branch target.
|
| Asm->bindCfgNodeLabel(getIndex());
|
| for (const Inst &I : Phis) {
|
| if (I.isDeleted())
|
| @@ -1094,33 +1070,33 @@ void CfgNode::emitIAS(Cfg *Func) const {
|
| return;
|
| }
|
|
|
| - // The remainder of the function handles emission with sandboxing.
|
| - // There are explicit bundle_lock regions delimited by bundle_lock
|
| - // and bundle_unlock instructions. All other instructions are
|
| - // treated as an implicit one-instruction bundle_lock region.
|
| - // Emission is done twice for each bundle_lock region. The first
|
| - // pass is a preliminary pass, after which we can figure out what
|
| - // nop padding is needed, then roll back, and make the final pass.
|
| + // The remainder of the function handles emission with sandboxing. There are
|
| + // explicit bundle_lock regions delimited by bundle_lock and bundle_unlock
|
| + // instructions. All other instructions are treated as an implicit
|
| + // one-instruction bundle_lock region. Emission is done twice for each
|
| + // bundle_lock region. The first pass is a preliminary pass, after which we
|
| + // can figure out what nop padding is needed, then roll back, and make the
|
| + // final pass.
|
| //
|
| - // Ideally, the first pass would be speculative and the second pass
|
| - // would only be done if nop padding were needed, but the structure
|
| - // of the integrated assembler makes it hard to roll back the state
|
| - // of label bindings, label links, and relocation fixups. Instead,
|
| - // the first pass just disables all mutation of that state.
|
| + // Ideally, the first pass would be speculative and the second pass would
|
| + // only be done if nop padding were needed, but the structure of the
|
| + // integrated assembler makes it hard to roll back the state of label
|
| + // bindings, label links, and relocation fixups. Instead, the first pass just
|
| + // disables all mutation of that state.
|
|
|
| BundleEmitHelper Helper(Asm, Func->getTarget(), Insts);
|
| InstList::const_iterator End = Insts.end();
|
| - // Retrying indicates that we had to roll back to the bundle_lock
|
| - // instruction to apply padding before the bundle_lock sequence.
|
| + // Retrying indicates that we had to roll back to the bundle_lock instruction
|
| + // to apply padding before the bundle_lock sequence.
|
| bool Retrying = false;
|
| for (InstList::const_iterator I = Insts.begin(); I != End; ++I) {
|
| if (I->isDeleted() || I->isRedundantAssign())
|
| continue;
|
|
|
| if (llvm::isa<InstBundleLock>(I)) {
|
| - // Set up the initial bundle_lock state. This should not happen
|
| - // while retrying, because the retry rolls back to the
|
| - // instruction following the bundle_lock instruction.
|
| + // Set up the initial bundle_lock state. This should not happen while
|
| + // retrying, because the retry rolls back to the instruction following
|
| + // the bundle_lock instruction.
|
| assert(!Retrying);
|
| Helper.enterBundleLock(I);
|
| continue;
|
| @@ -1131,16 +1107,16 @@ void CfgNode::emitIAS(Cfg *Func) const {
|
| if (Retrying) {
|
| // Make sure all instructions are in the same bundle.
|
| assert(Helper.isSameBundle());
|
| - // If align_to_end is specified, make sure the next
|
| - // instruction begins the bundle.
|
| + // If align_to_end is specified, make sure the next instruction begins
|
| + // the bundle.
|
| assert(!Helper.isAlignToEnd() || Helper.getPostAlignment() == 0);
|
| Helper.leaveBundleLockRegion();
|
| Retrying = false;
|
| } else {
|
| // This is the first pass, so roll back for the retry pass.
|
| Helper.rollback();
|
| - // Pad to the next bundle if the instruction sequence crossed
|
| - // a bundle boundary.
|
| + // Pad to the next bundle if the instruction sequence crossed a bundle
|
| + // boundary.
|
| Helper.padToNextBundle();
|
| // Insert additional padding to make AlignToEnd work.
|
| Helper.padForAlignToEnd();
|
| @@ -1171,8 +1147,8 @@ void CfgNode::emitIAS(Cfg *Func) const {
|
| }
|
| }
|
|
|
| - // Don't allow bundle locking across basic blocks, to keep the
|
| - // backtracking mechanism simple.
|
| + // Don't allow bundle locking across basic blocks, to keep the backtracking
|
| + // mechanism simple.
|
| assert(!Helper.isInBundleLockRegion());
|
| assert(!Retrying);
|
| }
|
|
|