diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 1e349e055ef989..29e0cd6fd968c1 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -7698,7 +7698,7 @@ class Compiler BasicBlock* exiting, LoopLocalOccurrences* loopLocals); bool optCanAndShouldChangeExitTest(GenTree* cond, bool dump); - bool optPrimaryIVHasNonLoopUses(unsigned lclNum, FlowGraphNaturalLoop* loop, LoopLocalOccurrences* loopLocals); + bool optLocalHasNonLoopUses(unsigned lclNum, FlowGraphNaturalLoop* loop, LoopLocalOccurrences* loopLocals); bool optWidenIVs(ScalarEvolutionContext& scevContext, FlowGraphNaturalLoop* loop, LoopLocalOccurrences* loopLocals); bool optWidenPrimaryIV(FlowGraphNaturalLoop* loop, diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp index 5f0dcc4eecd528..89d8c979a7685d 100644 --- a/src/coreclr/jit/inductionvariableopts.cpp +++ b/src/coreclr/jit/inductionvariableopts.cpp @@ -1093,7 +1093,7 @@ bool Compiler::optMakeExitTestDownwardsCounted(ScalarEvolutionContext& scevConte unsigned candidateLclNum = stmt->GetRootNode()->AsLclVarCommon()->GetLclNum(); - if (optPrimaryIVHasNonLoopUses(candidateLclNum, loop, loopLocals)) + if (optLocalHasNonLoopUses(candidateLclNum, loop, loopLocals)) { continue; } @@ -1258,20 +1258,19 @@ bool Compiler::optCanAndShouldChangeExitTest(GenTree* cond, bool dump) } //------------------------------------------------------------------------ -// optPrimaryIVHasNonLoopUses: -// Check if a primary IV may have uses of the primary IV that we do not -// reason about. +// optLocalHasNonLoopUses: +// Check if a loop may have uses of a local that we do not reason about. // // Parameters: -// lclNum - The primary IV +// lclNum - The local // loop - The loop // loopLocals - Data structure tracking local uses // // Returns: -// True if the primary IV may have non-loop uses (or if it is a field with -// uses of the parent struct). +// True if the local may have non-loop uses (or if it is a field with uses of +// the parent struct). // -bool Compiler::optPrimaryIVHasNonLoopUses(unsigned lclNum, FlowGraphNaturalLoop* loop, LoopLocalOccurrences* loopLocals) +bool Compiler::optLocalHasNonLoopUses(unsigned lclNum, FlowGraphNaturalLoop* loop, LoopLocalOccurrences* loopLocals) { LclVarDsc* varDsc = lvaGetDesc(lclNum); if (varDsc->lvIsStructField && loopLocals->HasAnyOccurrences(loop, varDsc->lvParentLcl)) @@ -1333,11 +1332,13 @@ class StrengthReductionContext SimplificationAssumptions m_simplAssumptions; ArrayStack m_cursors1; ArrayStack m_cursors2; + ArrayStack m_intermediateIVStores; void InitializeSimplificationAssumptions(); bool InitializeCursors(GenTreeLclVarCommon* primaryIVLcl, ScevAddRec* primaryIV); bool IsUseExpectedToBeRemoved(BasicBlock* block, Statement* stmt, GenTreeLclVarCommon* tree); void AdvanceCursors(ArrayStack* cursors, ArrayStack* nextCursors); + void ExpandStoredCursors(ArrayStack* cursors, ArrayStack* otherCursors); bool CheckAdvancedCursors(ArrayStack* cursors, ScevAddRec** nextIV); bool StaysWithinManagedObject(ArrayStack* cursors, ScevAddRec* addRec); bool TryReplaceUsesWithNewPrimaryIV(ArrayStack* cursors, ScevAddRec* iv); @@ -1345,6 +1346,7 @@ class StrengthReductionContext BasicBlock* FindPostUseUpdateInsertionPoint(ArrayStack* cursors, BasicBlock* backEdgeDominator, Statement** afterStmt); + Statement* LatestStatement(Statement* stmt1, Statement* stmt2); bool InsertionPointPostDominatesUses(BasicBlock* insertionPoint, ArrayStack* cursors); bool StressProfitability() @@ -1364,6 +1366,7 @@ class StrengthReductionContext , m_backEdgeBounds(comp->getAllocator(CMK_LoopIVOpts)) , m_cursors1(comp->getAllocator(CMK_LoopIVOpts)) , m_cursors2(comp->getAllocator(CMK_LoopIVOpts)) + , m_intermediateIVStores(comp->getAllocator(CMK_LoopIVOpts)) { } @@ -1436,7 +1439,7 @@ bool StrengthReductionContext::TryStrengthReduce() continue; } - if (m_comp->optPrimaryIVHasNonLoopUses(primaryIVLcl->GetLclNum(), m_loop, &m_loopLocals)) + if (m_comp->optLocalHasNonLoopUses(primaryIVLcl->GetLclNum(), m_loop, &m_loopLocals)) { // We won't be able to remove this primary IV JITDUMP(" Has non-loop uses\n"); @@ -1469,6 +1472,7 @@ bool StrengthReductionContext::TryStrengthReduce() { break; } + assert(nextIV != nullptr); if (varTypeIsGC(nextIV->Type) && !StaysWithinManagedObject(nextCursors, nextIV)) @@ -1479,6 +1483,8 @@ bool StrengthReductionContext::TryStrengthReduce() break; } + ExpandStoredCursors(nextCursors, cursors); + derivedLevel++; std::swap(cursors, nextCursors); currentIV = nextIV; @@ -1589,6 +1595,7 @@ bool StrengthReductionContext::InitializeCursors(GenTreeLclVarCommon* primaryIVL { m_cursors1.Reset(); m_cursors2.Reset(); + m_intermediateIVStores.Reset(); auto visitor = [=](BasicBlock* block, Statement* stmt, GenTreeLclVarCommon* tree) { if (IsUseExpectedToBeRemoved(block, stmt, tree)) @@ -1636,6 +1643,8 @@ bool StrengthReductionContext::InitializeCursors(GenTreeLclVarCommon* primaryIVL return false; } + ExpandStoredCursors(&m_cursors1, &m_cursors2); + JITDUMP(" Found %d cursors using primary IV V%02u\n", m_cursors1.Height(), primaryIVLcl->GetLclNum()); #ifdef DEBUG @@ -1755,9 +1764,6 @@ void StrengthReductionContext::AdvanceCursors(ArrayStack* cursors, A break; } - // TODO-CQ: If this is now the source to a store, we can - // look for uses of the LHS local and add those as cursors - // as well. Scev* parentIV = m_scevContext.Analyze(nextCursor.Block, nextCursor.Tree); if (parentIV == nullptr) { @@ -1798,6 +1804,116 @@ void StrengthReductionContext::AdvanceCursors(ArrayStack* cursors, A #endif } +//------------------------------------------------------------------------ +// ExpandStoredCursors: For every cursor that is the source to a store, expand +// the sets of cursors to contain the destination local's uses if possible. +// +// Parameters: +// cursors - [in, out] List of current cursors to expand. +// otherCursors - [in, out] The other list of cursors. +// +void StrengthReductionContext::ExpandStoredCursors(ArrayStack* cursors, + ArrayStack* otherCursors) +{ + for (int i = 0; i < cursors->Height(); i++) + { + while (true) + { + CursorInfo* cursor = &cursors->BottomRef(i); + GenTree* cur = cursor->Tree; + + GenTree* parent = cur->gtGetParent(nullptr); + if ((parent == nullptr) || (parent->OperIs(GT_COMMA) && (parent->gtGetOp1() == cur))) + { + break; + } + + if (parent->OperIs(GT_STORE_LCL_VAR)) + { + GenTreeLclVarCommon* storedLcl = parent->AsLclVarCommon(); + if ((storedLcl->Data() == cur) && ((cur->gtFlags & GTF_SIDE_EFFECT) == 0) && + storedLcl->HasSsaIdentity() && + !m_comp->optLocalHasNonLoopUses(storedLcl->GetLclNum(), m_loop, &m_loopLocals)) + { + int numCreated = 0; + ScevAddRec* cursorIV = cursor->IV; + BasicBlock* cursorBlock = cursor->Block; + Statement* cursorStmt = cursor->Stmt; + cursor = nullptr; // Cannot use this below since we may add elements to "cursors" here. + + auto createExtraCursor = [=, &numCreated](BasicBlock* block, Statement* stmt, + GenTreeLclVarCommon* use) { + if (use == parent) + { + return true; + } + + if (!use->OperIs(GT_LCL_VAR) || (use->GetSsaNum() != storedLcl->GetSsaNum())) + { + return false; + } + + Scev* iv = m_scevContext.Analyze(block, use); + if (iv == nullptr) + { + return false; + } + + iv = m_scevContext.Simplify(iv, m_simplAssumptions); + assert(iv != nullptr); + if (!Scev::Equals(iv, cursorIV)) + { + return false; + } + + // Note: cannot use "cursor" after this point. + cursors->Emplace(block, stmt, use, cursorIV); + otherCursors->Emplace(block, stmt, use, cursorIV); + numCreated++; + return true; + }; + + if (m_loopLocals.VisitOccurrences(m_loop, storedLcl->GetLclNum(), createExtraCursor)) + { + JITDUMP( + " [%06u] was the data of store [%06u]; expanded to %d new cursors, and will replace with a store of 0\n", + Compiler::dspTreeID(cur), Compiler::dspTreeID(parent), numCreated); + // We created cursors for all uses. Remove the IV that + // was feeding into the store from the list. + m_intermediateIVStores.Emplace(cursorBlock, cursorStmt, parent, nullptr); + std::swap(cursors->BottomRef(i), cursors->TopRef(0)); + std::swap(otherCursors->BottomRef(i), otherCursors->TopRef(0)); + cursors->Pop(); + otherCursors->Pop(); + i--; + break; + } + + cursors->Pop(numCreated); + otherCursors->Pop(numCreated); + } + + break; + } + + Scev* parentIV = m_scevContext.Analyze(cursor->Block, parent); + if (parentIV == nullptr) + { + break; + } + + parentIV = m_scevContext.Simplify(parentIV, m_simplAssumptions); + assert(parentIV != nullptr); + if (!Scev::Equals(parentIV, cursor->IV)) + { + break; + } + + cursor->Tree = parent; + } + } +} + //------------------------------------------------------------------------ // CheckAdvancedCursors: Check whether the specified advanced cursors still // represent a valid set of cursors to introduce a new primary IV for. @@ -1885,10 +2001,15 @@ bool StrengthReductionContext::StaysWithinManagedObject(ArrayStack* for (int i = 0; i < cursors->Height(); i++) { CursorInfo& cursor = cursors->BottomRef(i); - GenTree* parent = cursor.Tree->gtGetParent(nullptr); - if ((parent != nullptr) && parent->OperIs(GT_ARR_ADDR)) + GenTree* cur = cursor.Tree; + while ((cur != nullptr) && !cur->OperIs(GT_ARR_ADDR)) + { + cur = cur->gtGetParent(nullptr); + } + + if (cur != nullptr) { - arrAddr = parent->AsArrAddr(); + arrAddr = cur->AsArrAddr(); break; } } @@ -2088,6 +2209,25 @@ bool StrengthReductionContext::TryReplaceUsesWithNewPrimaryIV(ArrayStackgtUpdateStmtSideEffects(cursor.Stmt); } + if (m_intermediateIVStores.Height() > 0) + { + JITDUMP(" Deleting stores of intermediate IVs\n"); + for (int i = 0; i < m_intermediateIVStores.Height(); i++) + { + CursorInfo& cursor = m_intermediateIVStores.BottomRef(i); + GenTreeLclVarCommon* store = cursor.Tree->AsLclVarCommon(); + JITDUMP(" Replacing [%06u] with a zero constant\n", Compiler::dspTreeID(store->Data())); + // We cannot remove these stores entirely as that will break + // downstream phases looking for SSA defs.. instead just replace + // the data with a zero and leave it up to backend liveness to + // remove that. + store->Data() = m_comp->gtNewZeroConNode(genActualType(store->Data())); + m_comp->gtSetStmtInfo(cursor.Stmt); + m_comp->fgSetStmtSeq(cursor.Stmt); + m_comp->gtUpdateStmtSideEffects(cursor.Stmt); + } + } + return true; } @@ -2181,36 +2321,7 @@ BasicBlock* StrengthReductionContext::FindPostUseUpdateInsertionPoint(ArrayStack Statement** afterStmt) { BitVecTraits poTraits = m_loop->GetDfsTree()->PostOrderTraits(); - -#ifdef DEBUG - // We will be relying on the fact that the cursors are ordered in a useful - // way here: loop locals are visited in post order within each basic block, - // meaning that "cursors" has the last uses first for each basic block. - // Assert that here. - - BitVec seenBlocks(BitVecOps::MakeEmpty(&poTraits)); - for (int i = 1; i < cursors->Height(); i++) - { - CursorInfo& prevCursor = cursors->BottomRef(i - 1); - CursorInfo& cursor = cursors->BottomRef(i); - - if (cursor.Block != prevCursor.Block) - { - assert(BitVecOps::TryAddElemD(&poTraits, seenBlocks, prevCursor.Block->bbPostorderNum)); - continue; - } - - Statement* curStmt = cursor.Stmt; - while ((curStmt != nullptr) && (curStmt != prevCursor.Stmt)) - { - curStmt = curStmt->GetNextStmt(); - } - - assert(curStmt == prevCursor.Stmt); - } -#endif - - BitVec blocksWithUses(BitVecOps::MakeEmpty(&poTraits)); + BitVec blocksWithUses(BitVecOps::MakeEmpty(&poTraits)); for (int i = 0; i < cursors->Height(); i++) { CursorInfo& cursor = cursors->BottomRef(i); @@ -2230,6 +2341,7 @@ BasicBlock* StrengthReductionContext::FindPostUseUpdateInsertionPoint(ArrayStack return nullptr; } + Statement* latestStmt = nullptr; for (int i = 0; i < cursors->Height(); i++) { CursorInfo& cursor = cursors->BottomRef(i); @@ -2238,19 +2350,67 @@ BasicBlock* StrengthReductionContext::FindPostUseUpdateInsertionPoint(ArrayStack continue; } - if (!InsertionPointPostDominatesUses(cursor.Block, cursors)) + if (latestStmt == nullptr) + { + latestStmt = cursor.Stmt; + } + else { - return nullptr; + latestStmt = LatestStatement(latestStmt, cursor.Stmt); } + } - *afterStmt = cursor.Stmt; - return cursor.Block; + assert(latestStmt != nullptr); + if (!InsertionPointPostDominatesUses(backEdgeDominator, cursors)) + { + return nullptr; } + + *afterStmt = latestStmt; + return backEdgeDominator; } return nullptr; } +//------------------------------------------------------------------------ +// LatestStatement: Given two statements in the same basic block, return the +// latter of the two. +// +// Parameters: +// stmt1 - First statement +// stmt2 - Second statement +// +// Returns: +// Latter of the statements. +// +Statement* StrengthReductionContext::LatestStatement(Statement* stmt1, Statement* stmt2) +{ + if (stmt1 == stmt2) + { + return stmt1; + } + + Statement* cursor1 = stmt1->GetNextStmt(); + Statement* cursor2 = stmt2->GetNextStmt(); + + while (true) + { + if ((cursor1 == stmt2) || (cursor2 == nullptr)) + { + return stmt2; + } + + if ((cursor2 == stmt1) || (cursor1 == nullptr)) + { + return stmt1; + } + + cursor1 = cursor1->GetNextStmt(); + cursor2 = cursor2->GetNextStmt(); + } +} + //------------------------------------------------------------------------ // InsertionPointPostDominatesUses: Check if a basic block post-dominates all // locations specified by the cursors. @@ -2317,7 +2477,7 @@ bool Compiler::optRemoveUnusedIVs(FlowGraphNaturalLoop* loop, LoopLocalOccurrenc unsigned lclNum = stmt->GetRootNode()->AsLclVarCommon()->GetLclNum(); JITDUMP(" V%02u", lclNum); - if (optPrimaryIVHasNonLoopUses(lclNum, loop, loopLocals)) + if (optLocalHasNonLoopUses(lclNum, loop, loopLocals)) { JITDUMP(" has non-loop uses, cannot remove\n"); continue;