This is an archive of the discontinued LLVM Phabricator instance.

Differential D97266

[mlir][Linalg] Retire hoistViewAllocOps.
ClosedPublic

Authored by nicolasvasilache on Feb 23 2021, 1:47 AM.

Details

Reviewers
pifon2a
bkramer
ftynse
ThomasRaoux
aartbik
Commits
rG8cf14b8deca1: [mlir][Linalg] Retire hoistViewAllocOps.
Summary

This transformation was only used for quick experimentation and is not general enough.
Retire it.

Diff Detail

Event Timeline

nicolasvasilache created this revision.Feb 23 2021, 1:47 AM
Herald added a reviewer: aartbik. · View Herald TranscriptFeb 23 2021, 1:47 AM
Herald added subscribers: mravishankar, teijeong, rdzhabarov and 15 others. · View Herald Transcript
nicolasvasilache requested review of this revision.Feb 23 2021, 1:47 AM
Herald added a project: Restricted Project. · View Herald TranscriptFeb 23 2021, 1:47 AM
Herald added subscribers: limo1996, stephenneuendorffer. · View Herald Transcript
Harbormaster completed remote builds in B90359: Diff 325714.Feb 23 2021, 2:43 AM
bkramer accepted this revision.Feb 23 2021, 3:42 AM

lg

This revision is now accepted and ready to land.Feb 23 2021, 3:42 AM
Closed by commit rG8cf14b8deca1: [mlir][Linalg] Retire hoistViewAllocOps. (authored by nicolasvasilache). · Explain WhyFeb 23 2021, 3:45 AM
This revision was automatically updated to reflect the committed changes.
nicolasvasilache added a commit: rG8cf14b8deca1: [mlir][Linalg] Retire hoistViewAllocOps..

Revision Contents

PathSize
mlir/
include/
mlir/
Dialect/
Linalg/
Transforms/
7 lines
lib/
Dialect/
Linalg/
Transforms/
1 line
47 lines
test/
Dialect/
Linalg/
430 lines
lib/
Transforms/
1 line
8 lines

Diff 325714

mlir/include/mlir/Dialect/Linalg/Transforms/Hoisting.h

Show All 10 Lines
namespace mlir { namespace mlir {
class FuncOp; class FuncOp;
struct LogicalResult; struct LogicalResult;
namespace linalg { namespace linalg {
class PadTensorOp; class PadTensorOp;
/// Hoist alloc/dealloc pairs and alloca op out of immediately enclosing
/// scf::ForOp if both conditions are true:
/// 1. All operands are defined outside the loop.
/// 2. All uses are ViewLikeOp or DeallocOp.
// TODO: generalize on a per-need basis.
void hoistViewAllocOps(FuncOp func);
/// Hoist vector.transfer_read/vector.transfer_write on buffers pairs out of /// Hoist vector.transfer_read/vector.transfer_write on buffers pairs out of
/// immediately enclosing scf::ForOp iteratively, if the following conditions /// immediately enclosing scf::ForOp iteratively, if the following conditions
/// are true: /// are true:
/// 1. The two ops access the same memref with the same indices. /// 1. The two ops access the same memref with the same indices.
/// 2. All operands are invariant under the enclosing scf::ForOp. /// 2. All operands are invariant under the enclosing scf::ForOp.
/// 3. No uses of the memref either dominate the transfer_read or are /// 3. No uses of the memref either dominate the transfer_read or are
/// dominated by the transfer_write (i.e. no aliasing between the write and /// dominated by the transfer_write (i.e. no aliasing between the write and
/// the read across the loop) /// the read across the loop)
▲ Show 20 Lines • Show All 58 Lines • Show Last 20 Lines

mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp

Show First 20 Lines • Show All 55 Lines • ▼ Show 20 Lines auto stage3Transforms = [&](Operation *op) {
if (lateCodegenStrategyOptions.enableLICM) { if (lateCodegenStrategyOptions.enableLICM) {
op->walk([&](LoopLikeOpInterface loopLike) { op->walk([&](LoopLikeOpInterface loopLike) {
LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop:\n")); LLVM_DEBUG(loopLike.print(llvm::dbgs() << "\nOriginal loop:\n"));
if (failed(moveLoopInvariantCode(loopLike))) if (failed(moveLoopInvariantCode(loopLike)))
llvm_unreachable("unexpected LICM failure"); llvm_unreachable("unexpected LICM failure");
}); });
} }
promoteSingleIterationLoops(cast<FuncOp>(op)); promoteSingleIterationLoops(cast<FuncOp>(op));
hoistViewAllocOps(cast<FuncOp>(op));
if (lateCodegenStrategyOptions.enableHoistRedundantVectorTransfers) if (lateCodegenStrategyOptions.enableHoistRedundantVectorTransfers)
hoistRedundantVectorTransfers(cast<FuncOp>(op)); hoistRedundantVectorTransfers(cast<FuncOp>(op));
if (lateCodegenStrategyOptions.enableHoistRedundantVectorTransfersOnTensor) if (lateCodegenStrategyOptions.enableHoistRedundantVectorTransfersOnTensor)
hoistRedundantVectorTransfersOnTensor(cast<FuncOp>(op)); hoistRedundantVectorTransfersOnTensor(cast<FuncOp>(op));
return success(); return success();
}; };
(void)linalg::applyStagedPatterns( (void)linalg::applyStagedPatterns(
func, stage1Patterns, std::move(stage2Patterns), stage3Transforms); func, stage1Patterns, std::move(stage2Patterns), stage3Transforms);
Show All 37 Lines

mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp

Show All 29 Lines
#define DEBUG_TYPE "linalg-hoisting" #define DEBUG_TYPE "linalg-hoisting"
#define DBGS() (dbgs() << '[' << DEBUG_TYPE << "] ") #define DBGS() (dbgs() << '[' << DEBUG_TYPE << "] ")
using namespace mlir; using namespace mlir;
using namespace mlir::linalg; using namespace mlir::linalg;
void mlir::linalg::hoistViewAllocOps(FuncOp func) {
bool changed = true;
while (changed) {
changed = false;
func.walk([&changed](Operation *op) {
if (!isa<AllocOp, AllocaOp, DeallocOp>(op))
return;
LLVM_DEBUG(DBGS() << "Candidate for hoisting: " << *op << "\n");
auto loop = dyn_cast<scf::ForOp>(op->getParentOp());
LLVM_DEBUG(DBGS() << "Parent op: " << *op->getParentOp() << "\n");
// Only hoist out of immediately enclosing scf::ForOp.
if (!loop)
return;
// If any operand is defined inside the loop don't hoist.
if (llvm::any_of(op->getOperands(), [&](Value v) {
return !loop.isDefinedOutsideOfLoop(v);
}))
return;
LLVM_DEBUG(DBGS() << "All operands defined outside \n");
// If alloc has other uses than ViewLikeOp and DeallocOp don't hoist.
Value v;
if (op->getNumResults() > 0) {
assert(op->getNumResults() == 1 && "Unexpected multi-result alloc");
v = op->getResult(0);
}
if (v && !llvm::all_of(v.getUses(), [&](OpOperand &operand) {
return isa<ViewLikeOpInterface, DeallocOp>(operand.getOwner());
})) {
LLVM_DEBUG(DBGS() << "Found non view-like or dealloc use: bail\n");
return;
}
// Move AllocOp before the loop.
if (isa<AllocOp, AllocaOp>(op))
(void)loop.moveOutOfLoop({op});
else // Move DeallocOp outside of the loop.
op->moveAfter(loop);
changed = true;
});
}
}
namespace { namespace {
/// Represents a unit of hoistable TransferWriteOp. This may comprise other /// Represents a unit of hoistable TransferWriteOp. This may comprise other
/// instructions that need to be hoisted too. /// instructions that need to be hoisted too.
struct HoistableWrite { struct HoistableWrite {
vector::TransferWriteOp transferWriteOp; vector::TransferWriteOp transferWriteOp;
SubTensorInsertOp subTensorInsertOp; SubTensorInsertOp subTensorInsertOp;
}; };
/// Represents a unit of hoistable TransferReadOp. This may comprise other /// Represents a unit of hoistable TransferReadOp. This may comprise other
▲ Show 20 Lines • Show All 697 Lines • Show Last 20 Lines

mlir/test/Dialect/Linalg/hoisting.mlir

// RUN: mlir-opt %s -test-linalg-hoisting=test-hoist-view-allocs -allow-unregistered-dialect -split-input-file | FileCheck %s // RUN: mlir-opt %s -test-linalg-hoisting=test-hoist-redundant-transfers -allow-unregistered-dialect -split-input-file | FileCheck %s
// RUN: mlir-opt %s -test-linalg-hoisting=test-hoist-redundant-transfers -allow-unregistered-dialect -split-input-file | FileCheck %s --check-prefix=VECTOR_TRANSFERS
// ----- // CHECK-LABEL: func @hoist_vector_transfer_pairs(
// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// CHECK-LABEL: func @hoist_allocs( // CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// CHECK-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// CHECK-SAME: %[[MEMREF4:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// CHECK-SAME: %[[MEMREF5:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// CHECK-SAME: %[[VAL:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[VAL:[a-zA-Z0-9]*]]: index,
// CHECK-SAME: %[[LB:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[LB:[a-zA-Z0-9]*]]: index,
// CHECK-SAME: %[[UB:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[UB:[a-zA-Z0-9]*]]: index,
// CHECK-SAME: %[[STEP:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[STEP:[a-zA-Z0-9]*]]: index,
// CHECK-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1 // CHECK-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1
func @hoist_allocs(%val: index, %lb : index, %ub : index, %step: index, %cmp: i1) {
// CHECK-DAG: alloca(%[[VAL]]) : memref<?xi8>
// CHECK-DAG: %[[A0:.*]] = alloc(%[[VAL]]) : memref<?xi8>
// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
// CHECK: alloca(%[[I]]) : memref<?xi8>
// CHECK: %[[A1:.*]] = alloc(%[[I]]) : memref<?xi8>
// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
// CHECK-DAG: alloca(%[[J]]) : memref<?xi8>
// CHECK-DAG: %[[A2:.*]] = alloc(%[[J]]) : memref<?xi8>
// CHECK: scf.for %[[K:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
scf.for %i = %lb to %ub step %step {
scf.for %j = %lb to %ub step %step {
scf.for %k = %lb to %ub step %step {
// Hoist allocs / deallocs outermost, keep view/subview below k.
%sa0 = alloca(%val) : memref<? x i8>
%a0 = alloc(%val) : memref<? x i8>
// CHECK: std.view %[[A0]][%[[LB]]][] : memref<?xi8> to memref<16xf32>
// CHECK: subview %[[A0]][0] [42] [1] : memref<?xi8> to memref<42xi8>
%v0 = view %a0[%lb][] : memref<? x i8> to memref<16 x f32>
%sv0 = subview %a0[0][42][1] : memref<? x i8> to memref<42 x i8>
dealloc %a0 : memref<? x i8>
// Hoist below i.
%sa1 = alloca(%i) : memref<? x i8>
%a1 = alloc(%i) : memref<? x i8>
dealloc %a1 : memref<? x i8>
// Hoist below j.
%sa2 = alloca(%j) : memref<? x i8>
%a2 = alloc(%j) : memref<? x i8>
dealloc %a2 : memref<? x i8>
// Don't hoist since k innermost.
// CHECK: alloca(%[[K]]) : memref<?xi8>
// CHECK: %[[A3:.*]] = alloc(%[[K]]) : memref<?xi8>
// CHECK: dealloc %[[A3]] : memref<?xi8>
%sa3 = alloca(%k) : memref<? x i8>
%a3 = alloc(%k) : memref<? x i8>
dealloc %a3 : memref<? x i8>
// No hoisting due to control flow.
// CHECK: scf.if %[[CMP]] {
// CHECK: alloca(%[[VAL]]) : memref<?xi8>
// CHECK: %[[A4:.*]] = alloc(%[[VAL]]) : memref<?xi8>
// CHECK: dealloc %[[A4]] : memref<?xi8>
scf.if %cmp {
%sa4 = alloca(%val) : memref<? x i8>
%a4 = alloc(%val) : memref<? x i8>
dealloc %a4 : memref<? x i8>
}
// No hoisting due to load/store.
// CHECK: %[[SA5:.*]] = alloca(%[[VAL]]) : memref<?xi8>
// CHECK: %[[A5:.*]] = alloc(%[[VAL]]) : memref<?xi8>
// CHECK: load %[[A5]][%[[LB]]] : memref<?xi8>
// CHECK: store %{{.*}}, %[[SA5]][%[[LB]]] : memref<?xi8>
// CHECK: dealloc %[[A5]] : memref<?xi8>
%sa5 = alloca(%val) : memref<? x i8>
%a5 = alloc(%val) : memref<? x i8>
%v5 = load %a5[%lb] : memref<? x i8>
store %v5, %sa5[%lb] : memref<? x i8>
dealloc %a5 : memref<? x i8>
}
}
}
// CHECK: }
// CHECK: dealloc %[[A2]] : memref<?xi8>
// CHECK: }
// CHECK: dealloc %[[A1]] : memref<?xi8>
// CHECK: }
// CHECK: dealloc %[[A0]] : memref<?xi8>
return
}
// -----
// VECTOR_TRANSFERS-LABEL: func @hoist_vector_transfer_pairs(
// VECTOR_TRANSFERS-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF4:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF5:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[VAL:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[LB:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[UB:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[STEP:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1
func @hoist_vector_transfer_pairs( func @hoist_vector_transfer_pairs(
%memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>, %memref2: memref<?x?xf32>, %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>, %memref2: memref<?x?xf32>,
%memref3: memref<?x?xf32>, %memref4: memref<?x?xf32>, %memref5: memref<?x?xf32>, %memref3: memref<?x?xf32>, %memref4: memref<?x?xf32>, %memref5: memref<?x?xf32>,
%val: index, %lb : index, %ub : index, %step: index, %cmp: i1) { %val: index, %lb : index, %ub : index, %step: index, %cmp: i1) {
%c0 = constant 0 : index %c0 = constant 0 : index
%cst = constant 0.0 : f32 %cst = constant 0.0 : f32
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32> // CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32>
// VECTOR_TRANSFERS: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> (vector<1xf32>) { // CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> (vector<1xf32>) {
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<2xf32> // CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> (vector<1xf32>, vector<2xf32>) { // CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> (vector<1xf32>, vector<2xf32>) {
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<3xf32> // CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<3xf32>
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<4xf32> // CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<4xf32>
// VECTOR_TRANSFERS: "some_crippling_use"(%[[MEMREF4]]) : (memref<?x?xf32>) -> () // CHECK: "some_crippling_use"(%[[MEMREF4]]) : (memref<?x?xf32>) -> ()
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<5xf32> // CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<5xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32> // CHECK: "some_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%[[MEMREF2]]) : (memref<?x?xf32>) -> vector<3xf32> // CHECK: "some_use"(%[[MEMREF2]]) : (memref<?x?xf32>) -> vector<3xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32> // CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<5xf32>) -> vector<5xf32> // CHECK: "some_use"(%{{.*}}) : (vector<5xf32>) -> vector<5xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<3xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<3xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<4xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<4xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<5xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<5xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: "some_crippling_use"(%[[MEMREF3]]) : (memref<?x?xf32>) -> () // CHECK: "some_crippling_use"(%[[MEMREF3]]) : (memref<?x?xf32>) -> ()
// VECTOR_TRANSFERS: scf.yield {{.*}} : vector<1xf32>, vector<2xf32> // CHECK: scf.yield {{.*}} : vector<1xf32>, vector<2xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<2xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<2xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: scf.yield {{.*}} : vector<1xf32> // CHECK: scf.yield {{.*}} : vector<1xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32>
scf.for %i = %lb to %ub step %step { scf.for %i = %lb to %ub step %step {
scf.for %j = %lb to %ub step %step { scf.for %j = %lb to %ub step %step {
%r0 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<1xf32> %r0 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<1xf32>
%r1 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32> %r1 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32>
%r2 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32> %r2 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>
%r3 = vector.transfer_read %memref3[%c0, %c0], %cst: memref<?x?xf32>, vector<4xf32> %r3 = vector.transfer_read %memref3[%c0, %c0], %cst: memref<?x?xf32>, vector<4xf32>
"some_crippling_use"(%memref4) : (memref<?x?xf32>) -> () "some_crippling_use"(%memref4) : (memref<?x?xf32>) -> ()
%r4 = vector.transfer_read %memref4[%c0, %c0], %cst: memref<?x?xf32>, vector<5xf32> %r4 = vector.transfer_read %memref4[%c0, %c0], %cst: memref<?x?xf32>, vector<5xf32>
Show All 14 Lines scf.for %j = %lb to %ub step %step {
"some_crippling_use"(%memref3) : (memref<?x?xf32>) -> () "some_crippling_use"(%memref3) : (memref<?x?xf32>) -> ()
} }
} }
return return
} }
// ----- // -----
// VECTOR_TRANSFERS-LABEL: func @hoist_vector_transfer_pairs_disjoint( // CHECK-LABEL: func @hoist_vector_transfer_pairs_disjoint(
// VECTOR_TRANSFERS-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>, // CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>, // CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>, // CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>, // CHECK-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[VAL:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[VAL:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[LB:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[LB:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[UB:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[UB:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[STEP:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[STEP:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[RANDOM:[a-zA-Z0-9]*]]: index, // CHECK-SAME: %[[RANDOM:[a-zA-Z0-9]*]]: index,
// VECTOR_TRANSFERS-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1 // CHECK-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1
func @hoist_vector_transfer_pairs_disjoint( func @hoist_vector_transfer_pairs_disjoint(
%memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>, %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>,
%memref2: memref<?x?xf32>, %memref3: memref<?x?xf32>, %val: index, %lb : index, %ub : index, %memref2: memref<?x?xf32>, %memref3: memref<?x?xf32>, %val: index, %lb : index, %ub : index,
%step: index, %random_index : index, %cmp: i1) { %step: index, %random_index : index, %cmp: i1) {
%c0 = constant 0 : index %c0 = constant 0 : index
%c1 = constant 1 : index %c1 = constant 1 : index
%c3 = constant 3 : index %c3 = constant 3 : index
%cst = constant 0.0 : f32 %cst = constant 0.0 : f32
// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32> // CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32> // CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32> // CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32> // CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>
// VECTOR_TRANSFERS: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> // CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
// VECTOR_TRANSFERS-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) { // CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
// VECTOR_TRANSFERS: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> // CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->
// VECTOR_TRANSFERS-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) { // CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32> // CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32> // CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32> // CHECK: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32> // CHECK: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32> // CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32> // CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32> // CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32> // CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>
scf.for %i = %lb to %ub step %step { scf.for %i = %lb to %ub step %step {
scf.for %j = %lb to %ub step %step { scf.for %j = %lb to %ub step %step {
%r00 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<2xf32> %r00 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<2xf32>
%r01 = vector.transfer_read %memref1[%c0, %c1], %cst: memref<?x?xf32>, vector<2xf32> %r01 = vector.transfer_read %memref1[%c0, %c1], %cst: memref<?x?xf32>, vector<2xf32>
%r20 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32> %r20 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>
%r21 = vector.transfer_read %memref2[%c0, %c3], %cst: memref<?x?xf32>, vector<3xf32> %r21 = vector.transfer_read %memref2[%c0, %c3], %cst: memref<?x?xf32>, vector<3xf32>
%r30 = vector.transfer_read %memref3[%c0, %random_index], %cst: memref<?x?xf32>, vector<4xf32> %r30 = vector.transfer_read %memref3[%c0, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
%r31 = vector.transfer_read %memref3[%c1, %random_index], %cst: memref<?x?xf32>, vector<4xf32> %r31 = vector.transfer_read %memref3[%c1, %random_index], %cst: memref<?x?xf32>, vector<4xf32>
Show All 17 Lines scf.for %j = %lb to %ub step %step {
vector.transfer_write %u11, %memref0[%random_index, %random_index] : vector<2xf32>, memref<?x?xf32> vector.transfer_write %u11, %memref0[%random_index, %random_index] : vector<2xf32>, memref<?x?xf32>
} }
} }
return return
} }
// ----- // -----
// VECTOR_TRANSFERS-LABEL: func @hoist_vector_transfer_pairs_tensor // CHECK-LABEL: func @hoist_vector_transfer_pairs_tensor
func @hoist_vector_transfer_pairs_tensor( func @hoist_vector_transfer_pairs_tensor(
%tensor0: tensor<?x?xf32>, %tensor1: tensor<?x?xf32>, %tensor2: tensor<?x?xf32>, %tensor0: tensor<?x?xf32>, %tensor1: tensor<?x?xf32>, %tensor2: tensor<?x?xf32>,
%tensor3: tensor<?x?xf32>, %tensor4: tensor<?x?xf32>, %tensor5: tensor<?x?xf32>, %tensor3: tensor<?x?xf32>, %tensor4: tensor<?x?xf32>, %tensor5: tensor<?x?xf32>,
%val: index, %lb : index, %ub : index, %step: index) -> %val: index, %lb : index, %ub : index, %step: index) ->
(tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>,
tensor<?x?xf32>, tensor<?x?xf32>) { tensor<?x?xf32>, tensor<?x?xf32>) {
%c0 = constant 0 : index %c0 = constant 0 : index
%cst = constant 0.0 : f32 %cst = constant 0.0 : f32
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<1xf32> // CHECK: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<1xf32>
// VECTOR_TRANSFERS: scf.for {{.*}} iter_args({{.*}}) -> // CHECK: scf.for {{.*}} iter_args({{.*}}) ->
// VECTOR_TRANSFERS-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32>) { // CHECK-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32>) {
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<2xf32> // CHECK: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: scf.for {{.*}} iter_args({{.*}}) -> // CHECK: scf.for {{.*}} iter_args({{.*}}) ->
// VECTOR_TRANSFERS-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<2xf32>, vector<1xf32>) { // CHECK-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<2xf32>, vector<1xf32>) {
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<4xf32> // CHECK: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<4xf32>
// VECTOR_TRANSFERS: "some_crippling_use"(%{{.*}}) : (tensor<?x?xf32>) -> () // CHECK: "some_crippling_use"(%{{.*}}) : (tensor<?x?xf32>) -> ()
// VECTOR_TRANSFERS: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<5xf32> // CHECK: vector.transfer_read %{{.*}} : tensor<?x?xf32>, vector<5xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32> // CHECK: "some_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (tensor<?x?xf32>) -> vector<3xf32> // CHECK: "some_use"(%{{.*}}) : (tensor<?x?xf32>) -> vector<3xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32> // CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<5xf32>) -> vector<5xf32> // CHECK: "some_use"(%{{.*}}) : (vector<5xf32>) -> vector<5xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<3xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<3xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<4xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<4xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<5xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<5xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: "some_crippling_use"(%{{.*}}) : (tensor<?x?xf32>) -> () // CHECK: "some_crippling_use"(%{{.*}}) : (tensor<?x?xf32>) -> ()
// VECTOR_TRANSFERS: scf.yield {{.*}} : // CHECK: scf.yield {{.*}} :
// VECTOR_TRANSFERS-SAME: tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<2xf32>, vector<1xf32> // CHECK-SAME: tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<2xf32>, vector<1xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<2xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<2xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: scf.yield {{.*}} : // CHECK: scf.yield {{.*}} :
// VECTOR_TRANSFERS-SAME: tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32> // CHECK-SAME: tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}} : vector<1xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, tensor<?x?xf32>
%0:6 = scf.for %i = %lb to %ub step %step %0:6 = scf.for %i = %lb to %ub step %step
iter_args(%arg0 = %tensor0, %arg1 = %tensor1, %arg2 = %tensor2, iter_args(%arg0 = %tensor0, %arg1 = %tensor1, %arg2 = %tensor2,
%arg3 = %tensor3, %arg4 = %tensor4, %arg5 = %tensor5) %arg3 = %tensor3, %arg4 = %tensor4, %arg5 = %tensor5)
-> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, -> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>,
tensor<?x?xf32>, tensor<?x?xf32>) { tensor<?x?xf32>, tensor<?x?xf32>) {
%1:6 = scf.for %j = %lb to %ub step %step %1:6 = scf.for %j = %lb to %ub step %step
iter_args(%arg6 = %arg0, %arg7 = %arg1, %arg8 = %arg2, iter_args(%arg6 = %arg0, %arg7 = %arg1, %arg8 = %arg2,
%arg9 = %arg3, %arg10 = %arg4, %arg11 = %arg5) %arg9 = %arg3, %arg10 = %arg4, %arg11 = %arg5)
Show All 29 Lines// CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, tensor<?x?xf32>
} }
return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5 : return %0#0, %0#1, %0#2, %0#3, %0#4, %0#5 :
tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>,
tensor<?x?xf32>, tensor<?x?xf32> tensor<?x?xf32>, tensor<?x?xf32>
} }
// ----- // -----
// VECTOR_TRANSFERS-LABEL: func @hoist_vector_transfer_pairs_disjoint_tensor( // CHECK-LABEL: func @hoist_vector_transfer_pairs_disjoint_tensor(
// VECTOR_TRANSFERS-SAME: %[[TENSOR0:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR0:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR1:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR1:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR2:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR2:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR3:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR3:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
func @hoist_vector_transfer_pairs_disjoint_tensor( func @hoist_vector_transfer_pairs_disjoint_tensor(
%tensor0: tensor<?x?xf32>, %tensor1: tensor<?x?xf32>, %tensor0: tensor<?x?xf32>, %tensor1: tensor<?x?xf32>,
%tensor2: tensor<?x?xf32>, %tensor3: tensor<?x?xf32>, %tensor2: tensor<?x?xf32>, %tensor3: tensor<?x?xf32>,
%val: index, %lb : index, %ub : index, %step: index, %val: index, %lb : index, %ub : index, %step: index,
%random_index : index) -> %random_index : index) ->
(tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) { (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) {
%c0 = constant 0 : index %c0 = constant 0 : index
%c1 = constant 1 : index %c1 = constant 1 : index
%c3 = constant 3 : index %c3 = constant 3 : index
%cst = constant 0.0 : f32 %cst = constant 0.0 : f32
// VECTOR_TRANSFERS: vector.transfer_read %[[TENSOR2]]{{.*}} : tensor<?x?xf32>, vector<3xf32> // CHECK: vector.transfer_read %[[TENSOR2]]{{.*}} : tensor<?x?xf32>, vector<3xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[TENSOR2]]{{.*}} : tensor<?x?xf32>, vector<3xf32> // CHECK: vector.transfer_read %[[TENSOR2]]{{.*}} : tensor<?x?xf32>, vector<3xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[TENSOR3]]{{.*}} : tensor<?x?xf32>, vector<4xf32> // CHECK: vector.transfer_read %[[TENSOR3]]{{.*}} : tensor<?x?xf32>, vector<4xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[TENSOR3]]{{.*}} : tensor<?x?xf32>, vector<4xf32> // CHECK: vector.transfer_read %[[TENSOR3]]{{.*}} : tensor<?x?xf32>, vector<4xf32>
// VECTOR_TRANSFERS: %[[R:.*]]:6 = scf.for {{.*}} iter_args({{.*}}) -> // CHECK: %[[R:.*]]:6 = scf.for {{.*}} iter_args({{.*}}) ->
// VECTOR_TRANSFERS-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) { // CHECK-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
// VECTOR_TRANSFERS: scf.for {{.*}} iter_args({{.*}}) -> // CHECK: scf.for {{.*}} iter_args({{.*}}) ->
// VECTOR_TRANSFERS-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) { // CHECK-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {
// VECTOR_TRANSFERS: vector.transfer_read %[[TENSOR1]]{{.*}} : tensor<?x?xf32>, vector<2xf32> // CHECK: vector.transfer_read %[[TENSOR1]]{{.*}} : tensor<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: vector.transfer_read %[[TENSOR1]]{{.*}} : tensor<?x?xf32>, vector<2xf32> // CHECK: vector.transfer_read %[[TENSOR1]]{{.*}} : tensor<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32> // CHECK: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32> // CHECK: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32> // CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32> // CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32> // CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %{{.*}}{{.*}} : vector<2xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %{{.*}}{{.*}} : vector<2xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %{{.*}}, %{{.*}}{{.*}} : vector<2xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %{{.*}}, %{{.*}}{{.*}} : vector<2xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: scf.yield {{.*}} : // CHECK: scf.yield {{.*}} :
// VECTOR_TRANSFERS-SAME: tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32> // CHECK-SAME: tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: scf.yield {{.*}} : // CHECK: scf.yield {{.*}} :
// VECTOR_TRANSFERS-SAME: tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32> // CHECK-SAME: tensor<?x?xf32>, tensor<?x?xf32>, vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>
// VECTOR_TRANSFERS: } // CHECK: }
// VECTOR_TRANSFERS: %[[TENSOR4:.*]] = vector.transfer_write %[[R]]#5, %[[TENSOR3]]{{.*}} : vector<4xf32>, tensor<?x?xf32> // CHECK: %[[TENSOR4:.*]] = vector.transfer_write %[[R]]#5, %[[TENSOR3]]{{.*}} : vector<4xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %[[R]]#4, %[[TENSOR4]]{{.*}} : vector<4xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %[[R]]#4, %[[TENSOR4]]{{.*}} : vector<4xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: %[[TENSOR5:.*]] = vector.transfer_write %[[R]]#3, %[[TENSOR2]]{{.*}} : vector<3xf32>, tensor<?x?xf32> // CHECK: %[[TENSOR5:.*]] = vector.transfer_write %[[R]]#3, %[[TENSOR2]]{{.*}} : vector<3xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: vector.transfer_write %[[R]]#2, %[[TENSOR5]]{{.*}} : vector<3xf32>, tensor<?x?xf32> // CHECK: vector.transfer_write %[[R]]#2, %[[TENSOR5]]{{.*}} : vector<3xf32>, tensor<?x?xf32>
%0:4 = scf.for %i = %lb to %ub step %step %0:4 = scf.for %i = %lb to %ub step %step
iter_args(%arg0 = %tensor0, %arg1 = %tensor1, %arg2 = %tensor2, iter_args(%arg0 = %tensor0, %arg1 = %tensor1, %arg2 = %tensor2,
%arg3 = %tensor3) %arg3 = %tensor3)
-> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) { -> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) {
%1:4 = scf.for %j = %lb to %ub step %step %1:4 = scf.for %j = %lb to %ub step %step
iter_args(%arg4 = %arg0, %arg5 = %arg1, %arg6 = %arg2, iter_args(%arg4 = %arg0, %arg5 = %arg1, %arg6 = %arg2,
%arg7 = %arg3) %arg7 = %arg3)
-> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) { -> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) {
Show All 25 Lines -> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) {
} }
scf.yield %1#0, %1#1, %1#2, %1#3 : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> scf.yield %1#0, %1#1, %1#2, %1#3 : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
} }
return %0#0, %0#1, %0#2, %0#3 : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> return %0#0, %0#1, %0#2, %0#3 : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
} }
// ----- // -----
// VECTOR_TRANSFERS-LABEL: func @hoist_vector_transfer_pairs_tensor_and_subtensors // CHECK-LABEL: func @hoist_vector_transfer_pairs_tensor_and_subtensors
// VECTOR_TRANSFERS-SAME: %[[TENSOR0:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR0:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR1:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR1:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR2:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR2:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR3:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR3:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR4:[a-zA-Z0-9]*]]: tensor<?x?xf32>, // CHECK-SAME: %[[TENSOR4:[a-zA-Z0-9]*]]: tensor<?x?xf32>,
// VECTOR_TRANSFERS-SAME: %[[TENSOR5:[a-zA-Z0-9]*]]: tensor<?x?xf32> // CHECK-SAME: %[[TENSOR5:[a-zA-Z0-9]*]]: tensor<?x?xf32>
func @hoist_vector_transfer_pairs_tensor_and_subtensors( func @hoist_vector_transfer_pairs_tensor_and_subtensors(
%tensor0: tensor<?x?xf32>, %tensor1: tensor<?x?xf32>, %tensor2: tensor<?x?xf32>, %tensor0: tensor<?x?xf32>, %tensor1: tensor<?x?xf32>, %tensor2: tensor<?x?xf32>,
%tensor3: tensor<?x?xf32>, %tensor4: tensor<?x?xf32>, %tensor5: tensor<?x?xf32>, %tensor3: tensor<?x?xf32>, %tensor4: tensor<?x?xf32>, %tensor5: tensor<?x?xf32>,
%val: index, %lb : index, %ub : index, %step: index) -> %val: index, %lb : index, %ub : index, %step: index) ->
( (
tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>//, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>//, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
) { ) {
%c0 = constant 0 : index %c0 = constant 0 : index
%cst = constant 0.0 : f32 %cst = constant 0.0 : f32
// VECTOR_TRANSFERS: scf.for %[[I:.*]] = {{.*}} iter_args( // CHECK: scf.for %[[I:.*]] = {{.*}} iter_args(
// VECTOR_TRANSFERS-SAME: %[[TENSOR0_ARG:[0-9a-zA-Z]+]] = %[[TENSOR0]], // CHECK-SAME: %[[TENSOR0_ARG:[0-9a-zA-Z]+]] = %[[TENSOR0]],
// VECTOR_TRANSFERS-SAME: %[[TENSOR1_ARG:[0-9a-zA-Z]+]] = %[[TENSOR1]], // CHECK-SAME: %[[TENSOR1_ARG:[0-9a-zA-Z]+]] = %[[TENSOR1]],
// VECTOR_TRANSFERS-SAME: %[[TENSOR2_ARG:[0-9a-zA-Z]+]] = %[[TENSOR2]] // CHECK-SAME: %[[TENSOR2_ARG:[0-9a-zA-Z]+]] = %[[TENSOR2]]
// VECTOR_TRANSFERS-SAME: ) -> // CHECK-SAME: ) ->
// VECTOR_TRANSFERS-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> // CHECK-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
%0:3 = scf.for %i = %lb to %ub step %step %0:3 = scf.for %i = %lb to %ub step %step
iter_args(%arg0 = %tensor0, %arg1 = %tensor1, %arg2 = %tensor2) iter_args(%arg0 = %tensor0, %arg1 = %tensor1, %arg2 = %tensor2)
-> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) { -> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) {
// Hoisted // Hoisted
// VECTOR_TRANSFERS: %[[ST0:.*]] = subtensor %[[TENSOR0_ARG]][%[[I]], %[[I]]]{{.*}}: tensor<?x?xf32> to tensor<?x?xf32> // CHECK: %[[ST0:.*]] = subtensor %[[TENSOR0_ARG]][%[[I]], %[[I]]]{{.*}}: tensor<?x?xf32> to tensor<?x?xf32>
// VECTOR_TRANSFERS: %[[V0:.*]] = vector.transfer_read %[[ST0]]{{.*}} : tensor<?x?xf32>, vector<1xf32> // CHECK: %[[V0:.*]] = vector.transfer_read %[[ST0]]{{.*}} : tensor<?x?xf32>, vector<1xf32>
// VECTOR_TRANSFERS: %[[R:.*]]:3 = scf.for %[[J:.*]] = {{.*}} iter_args( // CHECK: %[[R:.*]]:3 = scf.for %[[J:.*]] = {{.*}} iter_args(
// VECTOR_TRANSFERS-SAME: %[[TENSOR1_ARG_L2:[0-9a-zA-Z]+]] = %[[TENSOR1_ARG]] // CHECK-SAME: %[[TENSOR1_ARG_L2:[0-9a-zA-Z]+]] = %[[TENSOR1_ARG]]
// VECTOR_TRANSFERS-SAME: %[[TENSOR2_ARG_L2:[0-9a-zA-Z]+]] = %[[TENSOR2_ARG]] // CHECK-SAME: %[[TENSOR2_ARG_L2:[0-9a-zA-Z]+]] = %[[TENSOR2_ARG]]
// VECTOR_TRANSFERS-SAME: %[[V0_ARG_L2:[0-9a-zA-Z]+]] = %[[V0]] // CHECK-SAME: %[[V0_ARG_L2:[0-9a-zA-Z]+]] = %[[V0]]
// VECTOR_TRANSFERS-SAME: ) -> // CHECK-SAME: ) ->
// VECTOR_TRANSFERS-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32> // CHECK-SAME: (tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32>
%1:3 = scf.for %j = %lb to %ub step %step %1:3 = scf.for %j = %lb to %ub step %step
iter_args(%arg6 = %arg0, %arg7 = %arg1, %arg8 = %arg2) iter_args(%arg6 = %arg0, %arg7 = %arg1, %arg8 = %arg2)
-> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) { -> (tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>) {
// Hoists. // Hoists.
%st0 = subtensor %arg6[%i, %i][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32> %st0 = subtensor %arg6[%i, %i][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
%r0 = vector.transfer_read %st0[%c0, %c0], %cst: tensor<?x?xf32>, vector<1xf32> %r0 = vector.transfer_read %st0[%c0, %c0], %cst: tensor<?x?xf32>, vector<1xf32>
// VECTOR_TRANSFERS: %[[ST1:.*]] = subtensor %[[TENSOR1_ARG_L2]][%[[J]],{{.*}}: tensor<?x?xf32> to tensor<?x?xf32> // CHECK: %[[ST1:.*]] = subtensor %[[TENSOR1_ARG_L2]][%[[J]],{{.*}}: tensor<?x?xf32> to tensor<?x?xf32>
// VECTOR_TRANSFERS: %[[V1:.*]] = vector.transfer_read %[[ST1]]{{.*}} : tensor<?x?xf32>, vector<2xf32> // CHECK: %[[V1:.*]] = vector.transfer_read %[[ST1]]{{.*}} : tensor<?x?xf32>, vector<2xf32>
// Does not hoist (subtensor depends on %j) // Does not hoist (subtensor depends on %j)
%st1 = subtensor %arg7[%j, %c0][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32> %st1 = subtensor %arg7[%j, %c0][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
%r1 = vector.transfer_read %st1[%c0, %c0], %cst: tensor<?x?xf32>, vector<2xf32> %r1 = vector.transfer_read %st1[%c0, %c0], %cst: tensor<?x?xf32>, vector<2xf32>
// VECTOR_TRANSFERS: %[[ST2:.*]] = subtensor %[[TENSOR2_ARG_L2]][%[[I]],{{.*}}: tensor<?x?xf32> to tensor<?x?xf32> // CHECK: %[[ST2:.*]] = subtensor %[[TENSOR2_ARG_L2]][%[[I]],{{.*}}: tensor<?x?xf32> to tensor<?x?xf32>
// VECTOR_TRANSFERS: %[[V2:.*]] = vector.transfer_read %[[ST2]]{{.*}} : tensor<?x?xf32>, vector<3xf32> // CHECK: %[[V2:.*]] = vector.transfer_read %[[ST2]]{{.*}} : tensor<?x?xf32>, vector<3xf32>
// Does not hoist, 2 subtensor %arg8. // Does not hoist, 2 subtensor %arg8.
%st2 = subtensor %arg8[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32> %st2 = subtensor %arg8[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
%r2 = vector.transfer_read %st2[%c0, %c0], %cst: tensor<?x?xf32>, vector<3xf32> %r2 = vector.transfer_read %st2[%c0, %c0], %cst: tensor<?x?xf32>, vector<3xf32>
// VECTOR_TRANSFERS: %[[U0:.*]] = "some_use"(%[[V0_ARG_L2]]) : (vector<1xf32>) -> vector<1xf32> // CHECK: %[[U0:.*]] = "some_use"(%[[V0_ARG_L2]]) : (vector<1xf32>) -> vector<1xf32>
// VECTOR_TRANSFERS: %[[U1:.*]] = "some_use"(%[[V1]]) : (vector<2xf32>) -> vector<2xf32> // CHECK: %[[U1:.*]] = "some_use"(%[[V1]]) : (vector<2xf32>) -> vector<2xf32>
// VECTOR_TRANSFERS: %[[U2:.*]] = "some_use"(%[[V2]]) : (vector<3xf32>) -> vector<3xf32> // CHECK: %[[U2:.*]] = "some_use"(%[[V2]]) : (vector<3xf32>) -> vector<3xf32>
%u0 = "some_use"(%r0) : (vector<1xf32>) -> vector<1xf32> %u0 = "some_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
%u1 = "some_use"(%r1) : (vector<2xf32>) -> vector<2xf32> %u1 = "some_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
%u2 = "some_use"(%r2) : (vector<3xf32>) -> vector<3xf32> %u2 = "some_use"(%r2) : (vector<3xf32>) -> vector<3xf32>
// Hoists // Hoists
%w0 = vector.transfer_write %u0, %st0[%c0, %c0] : vector<1xf32>, tensor<?x?xf32> %w0 = vector.transfer_write %u0, %st0[%c0, %c0] : vector<1xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS-DAG: %[[STI1:.*]] = vector.transfer_write %[[U1]], %{{.*}} : vector<2xf32>, tensor<?x?xf32> // CHECK-DAG: %[[STI1:.*]] = vector.transfer_write %[[U1]], %{{.*}} : vector<2xf32>, tensor<?x?xf32>
// Does not hoist (associated subtensor depends on %j). // Does not hoist (associated subtensor depends on %j).
%w1 = vector.transfer_write %u1, %st1[%i, %i] : vector<2xf32>, tensor<?x?xf32> %w1 = vector.transfer_write %u1, %st1[%i, %i] : vector<2xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS-DAG: %[[STI2:.*]] = vector.transfer_write %[[U2]], %{{.*}} : vector<3xf32>, tensor<?x?xf32> // CHECK-DAG: %[[STI2:.*]] = vector.transfer_write %[[U2]], %{{.*}} : vector<3xf32>, tensor<?x?xf32>
// Does not hoist, 2 subtensor / subtensor_insert for %arg8. // Does not hoist, 2 subtensor / subtensor_insert for %arg8.
%w2 = vector.transfer_write %u2, %st2[%c0, %c0] : vector<3xf32>, tensor<?x?xf32> %w2 = vector.transfer_write %u2, %st2[%c0, %c0] : vector<3xf32>, tensor<?x?xf32>
// Hoists. // Hoists.
%sti0 = subtensor_insert %w0 into %arg6[%i, %i][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32> %sti0 = subtensor_insert %w0 into %arg6[%i, %i][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32>
// VECTOR_TRANSFERS-DAG: subtensor_insert %[[STI1]] into %[[TENSOR1_ARG_L2]][%[[J]],{{.*}}: tensor<?x?xf32> into tensor<?x?xf32> // CHECK-DAG: subtensor_insert %[[STI1]] into %[[TENSOR1_ARG_L2]][%[[J]],{{.*}}: tensor<?x?xf32> into tensor<?x?xf32>
// Does not hoist (depends on %j). // Does not hoist (depends on %j).
%sti1 = subtensor_insert %w1 into %arg7[%j, %c0][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32> %sti1 = subtensor_insert %w1 into %arg7[%j, %c0][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32>
// VECTOR_TRANSFERS-DAG: subtensor_insert %[[STI2]] into %[[TENSOR2_ARG_L2]][%[[I]],{{.*}}: tensor<?x?xf32> into tensor<?x?xf32> // CHECK-DAG: subtensor_insert %[[STI2]] into %[[TENSOR2_ARG_L2]][%[[I]],{{.*}}: tensor<?x?xf32> into tensor<?x?xf32>
// Does not hoist, 2 subtensor / subtensor_insert for %arg8. // Does not hoist, 2 subtensor / subtensor_insert for %arg8.
%sti2 = subtensor_insert %w2 into %arg8[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32> %sti2 = subtensor_insert %w2 into %arg8[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32>
%st22 = subtensor %sti2[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32> %st22 = subtensor %sti2[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
%sti22 = subtensor_insert %st22 into %arg8[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32> %sti22 = subtensor_insert %st22 into %arg8[%i, %c0][%step, %step][1, 1] : tensor<?x?xf32> into tensor<?x?xf32>
// VECTOR_TRANSFERS: scf.yield {{.*}} : tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32> // CHECK: scf.yield {{.*}} : tensor<?x?xf32>, tensor<?x?xf32>, vector<1xf32>
// VECTOR_TRANSFERS: } // CHECK: }
scf.yield %sti0, %sti1, %sti22: scf.yield %sti0, %sti1, %sti22:
tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
} }
// Hoisted // Hoisted
// VECTOR_TRANSFERS: %[[STI0:.*]] = vector.transfer_write %[[R]]#2, %[[ST0]]{{.*}} : vector<1xf32>, tensor<?x?xf32> // CHECK: %[[STI0:.*]] = vector.transfer_write %[[R]]#2, %[[ST0]]{{.*}} : vector<1xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: subtensor_insert %[[STI0]] into %[[TENSOR0_ARG]][%[[I]], %[[I]]]{{.*}} : tensor<?x?xf32> into tensor<?x?xf32> // CHECK: subtensor_insert %[[STI0]] into %[[TENSOR0_ARG]][%[[I]], %[[I]]]{{.*}} : tensor<?x?xf32> into tensor<?x?xf32>
// VECTOR_TRANSFERS: scf.yield {{.*}} : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> // CHECK: scf.yield {{.*}} : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
scf.yield %1#0, %1#1, %1#2 : scf.yield %1#0, %1#1, %1#2 :
tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
// VECTOR_TRANSFERS: } // CHECK: }
} }
return %0#0, %0#1, %0#2 : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32> return %0#0, %0#1, %0#2 : tensor<?x?xf32>, tensor<?x?xf32>, tensor<?x?xf32>
} }

mlir/test/lib/Transforms/TestConvVectorization.cpp

Show First 20 Lines • Show All 68 Lines • ▼ Show 20 Linesvoid TestConvVectorization::runOnOperation() {
auto stage3Transforms = [](Operation *op) { auto stage3Transforms = [](Operation *op) {
PassManager pm(op->getContext()); PassManager pm(op->getContext());
pm.addPass(createLoopInvariantCodeMotionPass()); pm.addPass(createLoopInvariantCodeMotionPass());
if (failed(pm.run(cast<ModuleOp>(op)))) if (failed(pm.run(cast<ModuleOp>(op))))
llvm_unreachable("Unexpected failure in cleanup pass pipeline."); llvm_unreachable("Unexpected failure in cleanup pass pipeline.");
op->walk([](FuncOp func) { op->walk([](FuncOp func) {
promoteSingleIterationLoops(func); promoteSingleIterationLoops(func);
linalg::hoistViewAllocOps(func);
linalg::hoistRedundantVectorTransfers(func); linalg::hoistRedundantVectorTransfers(func);
}); });
return success(); return success();
}; };
(void)linalg::applyStagedPatterns(module, frozenStage1Patterns, (void)linalg::applyStagedPatterns(module, frozenStage1Patterns,
std::move(stage2Patterns), std::move(stage2Patterns),
stage3Transforms); stage3Transforms);
▲ Show 20 Lines • Show All 49 Lines • Show Last 20 Lines

mlir/test/lib/Transforms/TestLinalgHoisting.cpp

Show All 23 Linesstruct TestLinalgHoisting
TestLinalgHoisting() = default; TestLinalgHoisting() = default;
TestLinalgHoisting(const TestLinalgHoisting &pass) {} TestLinalgHoisting(const TestLinalgHoisting &pass) {}
void getDependentDialects(DialectRegistry &registry) const override { void getDependentDialects(DialectRegistry &registry) const override {
registry.insert<AffineDialect>(); registry.insert<AffineDialect>();
} }
void runOnFunction() override; void runOnFunction() override;
Option<bool> testHoistViewAllocs{
*this, "test-hoist-view-allocs",
llvm::cl::desc("Test hoisting alloc used by view"),
llvm::cl::init(false)};
Option<bool> testHoistRedundantTransfers{ Option<bool> testHoistRedundantTransfers{
*this, "test-hoist-redundant-transfers", *this, "test-hoist-redundant-transfers",
llvm::cl::desc("Test hoisting transfer_read/transfer_write pairs"), llvm::cl::desc("Test hoisting transfer_read/transfer_write pairs"),
llvm::cl::init(false)}; llvm::cl::init(false)};
}; };
} // end anonymous namespace } // end anonymous namespace
void TestLinalgHoisting::runOnFunction() { void TestLinalgHoisting::runOnFunction() {
if (testHoistViewAllocs) {
hoistViewAllocOps(getFunction());
return;
}
if (testHoistRedundantTransfers) { if (testHoistRedundantTransfers) {
hoistRedundantVectorTransfers(getFunction()); hoistRedundantVectorTransfers(getFunction());
hoistRedundantVectorTransfersOnTensor(getFunction()); hoistRedundantVectorTransfersOnTensor(getFunction());
return; return;
} }
} }
namespace mlir { namespace mlir {
namespace test { namespace test {
void registerTestLinalgHoisting() { void registerTestLinalgHoisting() {
PassRegistration<TestLinalgHoisting> testTestLinalgHoistingPass( PassRegistration<TestLinalgHoisting> testTestLinalgHoistingPass(
"test-linalg-hoisting", "Test Linalg hoisting functions."); "test-linalg-hoisting", "Test Linalg hoisting functions.");
} }
} // namespace test } // namespace test
} // namespace mlir } // namespace mlir