diff --git a/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.cpp b/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.cpp index a03c52f667cf..0ad689ee35d5 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.cpp +++ b/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.cpp @@ -655,6 +655,10 @@ void LoopOp::build(mlir::OpBuilder &b, mlir::OperationState &result, } } +ValueRange LoopOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? getOperation()->getResults() : ValueRange(); +} + void LoopOp::getSuccessorRegions(RegionBranchPoint point, SmallVectorImpl ®ions) { // If the predecessor is the GenericOp, branch into the body. @@ -664,7 +668,7 @@ void LoopOp::getSuccessorRegions(RegionBranchPoint point, } // Otherwise, the region branches back to the parent operation. - regions.push_back(RegionSuccessor(getOperation(), getResults())); + regions.push_back(RegionSuccessor::parent()); } SmallVector LoopOp::getInitTiedResultIndices() { diff --git a/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.td b/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.td index 38abfb594a7c..92ff5d52bbb4 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.td +++ b/compiler/src/iree/compiler/Codegen/Dialect/PCF/IR/PCFOps.td @@ -259,7 +259,7 @@ def LoopOp : PCF_Op<"loop", [ AutomaticAllocationScope, RecursiveMemoryEffects, DeclareOpInterfaceMethods, - DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"mlir::iree_compiler::IREE::PCF::ReturnOp"> ]> { let summary = [{ diff --git a/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.cpp b/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.cpp index 445ba69e2870..5a20ae1a03a6 100644 --- a/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.cpp +++ b/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.cpp @@ -1013,15 +1013,19 @@ void DeviceMemoizeOp::build(OpBuilder &builder, OperationState &state, state.addRegion(); } +ValueRange DeviceMemoizeOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? getOperation()->getResults() : ValueRange(); +} + void DeviceMemoizeOp::getSuccessorRegions( RegionBranchPoint point, SmallVectorImpl ®ions) { // Unconditional control flow into the region and back to the parent, so // return the correct RegionSuccessor purely based on the index being None or // 0. if (!point.isParent()) { - regions.push_back(RegionSuccessor({})); + regions.push_back(RegionSuccessor::parent()); } else { - regions.push_back(RegionSuccessor(&getBody(), getBody().getArguments())); + regions.push_back(RegionSuccessor(&getBody())); } } diff --git a/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.td b/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.td index 88039925d097..41b3fdd6f165 100644 --- a/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.td +++ b/compiler/src/iree/compiler/Dialect/HAL/IR/HALOps.td @@ -571,7 +571,7 @@ def HAL_DispatchExternOp : HAL_PureOp<"dispatch.extern", [ def HAL_DeviceMemoizeOp : HAL_Op<"device.memoize", [ RecursiveMemoryEffects, - DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"IREE::HAL::ReturnOp">, ]> { let summary = [{Memoizes resources for a particular device and queue affinity.}]; diff --git a/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.cpp b/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.cpp index d28ce500af6c..b8e60af6aa01 100644 --- a/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.cpp +++ b/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.cpp @@ -3413,15 +3413,20 @@ AsyncExecuteOp::getEntrySuccessorOperands(RegionSuccessor successor) { return getResourceOperands(); } +ValueRange AsyncExecuteOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? ValueRange(getResults()) + : ValueRange(getBodyRegion().getArguments()); +} + void AsyncExecuteOp::getSuccessorRegions( RegionBranchPoint point, SmallVectorImpl ®ions) { // Unconditional control flow into the region and back to the parent, so // return the correct RegionSuccessor purely based on the index being None or // 0. if (!point.isParent()) { - regions.push_back(RegionSuccessor(getOperation(), getResults())); + regions.push_back(RegionSuccessor::parent()); } else { - regions.push_back(RegionSuccessor(&getBody(), getBody().getArguments())); + regions.push_back(RegionSuccessor(&getBody())); } } @@ -3570,15 +3575,20 @@ AsyncConcurrentOp::getEntrySuccessorOperands(RegionSuccessor successor) { return getResourceOperands(); } +ValueRange AsyncConcurrentOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? ValueRange(getResults()) + : ValueRange(getBodyRegion().getArguments()); +} + void AsyncConcurrentOp::getSuccessorRegions( RegionBranchPoint point, SmallVectorImpl ®ions) { // Unconditional control flow into the region and back to the parent, so // return the correct RegionSuccessor purely based on the index being None or // 0. if (!point.isParent()) { - regions.push_back(RegionSuccessor(getOperation(), getResults())); + regions.push_back(RegionSuccessor::parent()); } else { - regions.push_back(RegionSuccessor(&getBody(), getBody().getArguments())); + regions.push_back(RegionSuccessor(&getBody())); } } @@ -4619,16 +4629,20 @@ CmdExecuteOp::getEntrySuccessorOperands(RegionSuccessor successor) { return getResourceOperands(); } +ValueRange CmdExecuteOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? ValueRange() + : ValueRange(getBodyRegion().getArguments()); +} + void CmdExecuteOp::getSuccessorRegions( RegionBranchPoint point, SmallVectorImpl ®ions) { // Unconditional control flow into the region and back to the parent, so // return the correct RegionSuccessor purely based on the index being None or // 0. if (!point.isParent()) { - regions.push_back( - RegionSuccessor(getOperation(), Operation::result_range(nullptr, 0))); + regions.push_back(RegionSuccessor::parent()); } else { - regions.push_back(RegionSuccessor(&getBody(), getBody().getArguments())); + regions.push_back(RegionSuccessor(&getBody())); } } @@ -4692,16 +4706,19 @@ LogicalResult CmdSerialOp::verify() { return success(); } +ValueRange CmdSerialOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? getOperation()->getResults() : ValueRange(); +} + void CmdSerialOp::getSuccessorRegions( RegionBranchPoint point, SmallVectorImpl ®ions) { // Unconditional control flow into the region and back to the parent, so // return the correct RegionSuccessor purely based on the index being None or // 0. if (!point.isParent()) { - regions.push_back( - RegionSuccessor(getOperation(), Operation::result_range(nullptr, 0))); + regions.push_back(RegionSuccessor::parent()); } else { - regions.push_back(RegionSuccessor(&getBody(), {})); + regions.push_back(RegionSuccessor(&getBody())); } } @@ -4719,16 +4736,19 @@ LogicalResult CmdConcurrentOp::verify() { return success(); } +ValueRange CmdConcurrentOp::getSuccessorInputs(RegionSuccessor successor) { + return successor.isParent() ? getOperation()->getResults() : ValueRange(); +} + void CmdConcurrentOp::getSuccessorRegions( RegionBranchPoint point, SmallVectorImpl ®ions) { // Unconditional control flow into the region and back to the parent, so // return the correct RegionSuccessor purely based on the index being None or // 0. if (!point.isParent()) { - regions.push_back( - RegionSuccessor(getOperation(), Operation::result_range(nullptr, 0))); + regions.push_back(RegionSuccessor::parent()); } else { - regions.push_back(RegionSuccessor(&getBody(), {})); + regions.push_back(RegionSuccessor(&getBody())); } } diff --git a/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.td b/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.td index 8a8469478932..46304d8e5266 100644 --- a/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.td +++ b/compiler/src/iree/compiler/Dialect/Stream/IR/StreamOps.td @@ -2797,6 +2797,7 @@ def Stream_AsyncExecuteOp : Stream_Op<"async.execute", [ RecursiveMemoryEffects, DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"IREE::Stream::YieldOp">, Stream_AffinityOp, @@ -2892,6 +2893,7 @@ def Stream_AsyncConcurrentOp : Stream_Op<"async.concurrent", [ RecursiveMemoryEffects, DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"IREE::Stream::YieldOp">, Stream_AffinityOp, @@ -3863,6 +3865,7 @@ def Stream_CmdExecuteOp : Stream_Op<"cmd.execute", [ RecursiveMemoryEffects, DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"IREE::Stream::YieldOp">, Stream_AffinityOp, @@ -3943,7 +3946,7 @@ def Stream_CmdSerialOp : Stream_Op<"cmd.serial", [ "IREE::Stream::CmdConcurrentOp", ]>, RecursiveMemoryEffects, - DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"IREE::Stream::YieldOp">, Stream_CmdPhaseOp, Stream_StreamableOp, @@ -4009,7 +4012,7 @@ def Stream_CmdConcurrentOp : Stream_Op<"cmd.concurrent", [ "IREE::Stream::CmdConcurrentOp", ]>, RecursiveMemoryEffects, - DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, SingleBlockImplicitTerminator<"IREE::Stream::YieldOp">, Stream_CmdPhaseOp, Stream_StreamableOp, diff --git a/compiler/src/iree/compiler/Dialect/Stream/Transforms/ScheduleAllocation.cpp b/compiler/src/iree/compiler/Dialect/Stream/Transforms/ScheduleAllocation.cpp index 294a4544568b..5dd7b0e75d80 100644 --- a/compiler/src/iree/compiler/Dialect/Stream/Transforms/ScheduleAllocation.cpp +++ b/compiler/src/iree/compiler/Dialect/Stream/Transforms/ScheduleAllocation.cpp @@ -1175,7 +1175,7 @@ static IREE::Stream::AffinityAttr findLocalValueAffinity(Value value) { auto terminatorOp = cast(block.getTerminator()); value = terminatorOp.getSuccessorOperands( - RegionSuccessor(definingOp, definingOp->getResults()))[resultIndex]; + RegionSuccessor::parent())[resultIndex]; } else if (auto tiedOp = dyn_cast(definingOp)) { // If the producer is tied then try to get the operand. @@ -1540,7 +1540,7 @@ static Value findTiedYieldResult(Value seedValue) { cast(seedValue.getParentRegion()->getParentOp()); SmallVector regions; regionOp.getSuccessorRegions(regionOp->getRegion(0), regions); - auto results = regions.front().getSuccessorInputs(); + auto results = regionOp.getSuccessorInputs(regions.front()); SmallVector worklist; worklist.push_back(seedValue); while (!worklist.empty()) { diff --git a/compiler/src/iree/compiler/Dialect/Util/Analysis/Explorer.cpp b/compiler/src/iree/compiler/Dialect/Util/Analysis/Explorer.cpp index eb7621c10bf1..c109158eb673 100644 --- a/compiler/src/iree/compiler/Dialect/Util/Analysis/Explorer.cpp +++ b/compiler/src/iree/compiler/Dialect/Util/Analysis/Explorer.cpp @@ -623,8 +623,7 @@ TraversalResult Explorer::walkReturnOperands(Operation *parentOp, return walkReturnOps(parentOp, [&](Operation *returnOp) { if (auto terminatorOp = dyn_cast(returnOp)) { - return fn(terminatorOp.getSuccessorOperands( - RegionSuccessor(parentOp, parentOp->getResults()))); + return fn(terminatorOp.getSuccessorOperands(RegionSuccessor::parent())); } else { return fn(returnOp->getOperands()); } @@ -1008,7 +1007,7 @@ TraversalResult Explorer::walkTransitiveUses(Value value, UseWalkFn fn, SmallVector entrySuccessors; regionOp.getSuccessorRegions(RegionBranchPoint::parent(), entrySuccessors); for (auto &entrySuccessor : entrySuccessors) { - auto successorInputs = entrySuccessor.getSuccessorInputs(); + auto successorInputs = regionOp.getSuccessorInputs(entrySuccessor); if (operandIdx >= successorInputs.size()) { // Implicit capture; argument has the same SSA value on the inside of // the region. Uses show up as normal so we ignore here. @@ -1030,9 +1029,8 @@ TraversalResult Explorer::walkTransitiveUses(Value value, UseWalkFn fn, // Move within/out-of a region. auto traverseRegionBranchOp = [&](RegionBranchTerminatorOpInterface branchOp, unsigned operandIdx) { - Operation *parentOp = branchOp.getOperation()->getParentOp(); - auto successorOperands = branchOp.getSuccessorOperands( - RegionSuccessor(parentOp, parentOp->getResults())); + auto successorOperands = + branchOp.getSuccessorOperands(RegionSuccessor::parent()); unsigned beginIdx = successorOperands.getBeginOperandIndex(); if (operandIdx < beginIdx || operandIdx >= beginIdx + successorOperands.size()) { diff --git a/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.cpp b/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.cpp index 96ca7590936f..98c51e010753 100644 --- a/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.cpp +++ b/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.cpp @@ -70,7 +70,7 @@ LogicalResult IntegerDivisibilityAnalysis::visitOperation( } void IntegerDivisibilityAnalysis::visitNonControlFlowArguments( - Operation *op, const RegionSuccessor &successor, + Operation *op, const RegionSuccessor &successor, ValueRange successorInputs, ArrayRef argLattices, unsigned firstIndex) { // Get the constant divisibility, or query the lattice for Values. auto getDivFromOfr = [&](std::optional ofr, Block *block, @@ -99,7 +99,7 @@ void IntegerDivisibilityAnalysis::visitNonControlFlowArguments( std::optional> steps = loop.getLoopSteps(); if (!ivs || !lbs || !steps) { return SparseForwardDataFlowAnalysis::visitNonControlFlowArguments( - op, successor, argLattices, firstIndex); + op, successor, successorInputs, argLattices, firstIndex); } for (auto [iv, lb, step] : llvm::zip_equal(*ivs, *lbs, *steps)) { IntegerDivisibilityLattice *ivEntry = getLatticeElement(iv); @@ -122,7 +122,7 @@ void IntegerDivisibilityAnalysis::visitNonControlFlowArguments( } return SparseForwardDataFlowAnalysis::visitNonControlFlowArguments( - op, successor, argLattices, firstIndex); + op, successor, successorInputs, argLattices, firstIndex); } } // namespace mlir::iree_compiler::IREE::Util diff --git a/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.h b/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.h index 72d3292596ea..0717d4dae7e4 100644 --- a/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.h +++ b/compiler/src/iree/compiler/Dialect/Util/Analysis/IntegerDivisibilityAnalysis.h @@ -42,6 +42,7 @@ class IntegerDivisibilityAnalysis /// on known loop bounds and steps. void visitNonControlFlowArguments( Operation *op, const RegionSuccessor &successor, + ValueRange successorInputs, ArrayRef argLattices, unsigned firstIndex) override; }; diff --git a/compiler/src/iree/compiler/Dialect/Util/Analysis/Position.cpp b/compiler/src/iree/compiler/Dialect/Util/Analysis/Position.cpp index d322763af384..3d26589dd449 100644 --- a/compiler/src/iree/compiler/Dialect/Util/Analysis/Position.cpp +++ b/compiler/src/iree/compiler/Dialect/Util/Analysis/Position.cpp @@ -96,7 +96,8 @@ SmallVector getReturnedValuePositions(Region ®ion) { regionOp.getSuccessorRegions(region, successors); for (auto &successor : successors) { if (successor.isParent()) { - return llvm::to_vector(getPositions(successor.getSuccessorInputs())); + return llvm::to_vector( + getPositions(regionOp.getSuccessorInputs(successor))); } } assert(false && "should have found a parent successor"); diff --git a/third_party/llvm-project b/third_party/llvm-project index 99323d97202f..03e1d32c7ccd 160000 --- a/third_party/llvm-project +++ b/third_party/llvm-project @@ -1 +1 @@ -Subproject commit 99323d97202fff34b8dcadbb7d9b6dd0394124e1 +Subproject commit 03e1d32c7ccd7f32c7e4efb1d3d98792b78de025 diff --git a/third_party/torch-mlir b/third_party/torch-mlir index ac33bab4f3c1..1b221cd8cc3f 160000 --- a/third_party/torch-mlir +++ b/third_party/torch-mlir @@ -1 +1 @@ -Subproject commit ac33bab4f3c1071d12b6c4c2f26bbcf87d1d2ab9 +Subproject commit 1b221cd8cc3f724d18a85933aaabc40ae3b4ed88