diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp --- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp @@ -294,22 +294,24 @@ nPar > 0 ? O(ivs) = fillOp.value() : O() = fillOp.value(); } +// Create a padded view into the given `input` tensor using the 'indices' +// to access the tensor. `skipPadding` lists the dimensions for which no padding +// is needed e.g. the non-spatial dimensions for convolutions. template -Value getConvOpInput(ConvOp convOp, StdIndexedValue im, - MutableArrayRef imIdx) { +Value getPaddedInput(Value input, ArrayRef indices, + ArrayRef skipPadding) { // TODO: add a level of indirection to linalg.generic. - if (!convOp.padding()) - return im(imIdx); + + IndexedValueType indexedInput(input); auto *context = ScopedContext::getContext(); Value zeroIndex = std_constant_index(0); SmallVector conds; SmallVector clampedImIdx; - for (auto iter : llvm::enumerate(imIdx)) { + for (auto iter : llvm::enumerate(indices)) { int idx = iter.index(); auto dim = iter.value(); - // Only need to iterate over the window dimensions. - if (idx == 0 || idx == static_cast(imIdx.size()) - 1) { + if (is_contained(skipPadding, idx)) { clampedImIdx.push_back(dim); continue; } @@ -322,7 +324,7 @@ conds.push_back(leftOutOfBound); else conds.push_back(conds.back() || leftOutOfBound); - Value rightBound = std_dim(convOp.input(), idx); + Value rightBound = std_dim(input, idx); conds.push_back(conds.back() || (sge(dim, rightBound))); // When padding is involved, the indices will only be shifted to negative, @@ -335,9 +337,9 @@ } auto &b = ScopedContext::getBuilderRef(); - Type type = convOp.input().getType().cast().getElementType(); + Type type = input.getType().cast().getElementType(); Value zero = std_constant(type, b.getZeroAttr(type)); - Value readInput = im(clampedImIdx); + Value readInput = indexedInput(clampedImIdx); return conds.empty() ? readInput : (Value)std_select(conds.back(), zero, readInput); } @@ -373,8 +375,10 @@ // which is not allowed by affine.load. Override to use an StdIndexedValue // when there is non-zero padding. if (hasPadding(convOp)) { - StdIndexedValue I(convOp.input()); - Value paddedInput = getConvOpInput(convOp, I, imIdx); + Value paddedInput = getPaddedInput( + convOp.input(), imIdx, + /* Only need to pad the window dimensions */ + {0, static_cast(imIdx.size()) - 1}); O(oIdx) += F(fIdx) * paddedInput; } else { IndexedValueType I(convOp.input()); @@ -382,14 +386,32 @@ } } +template +static bool hasPadding(PoolingOp poolingOp) { + for (unsigned i = 0, e = poolingOp.getNumWindowLoops(); i < e; ++i) { + if (poolingOp.getLowPad(i) > 0 || poolingOp.getHighPad(i) > 0) + return true; + } + return false; +} + +template +static Value getPoolingInput(PoolingOp op, ArrayRef inputIndices) { + if (hasPadding(op)) { + return getPaddedInput(op.input(), inputIndices, + /*Pad every dimension*/ {}); + } + IndexedValueType input(op.input()); + return input(inputIndices); +} + template void emitScalarImplementation(ArrayRef allIvs, PoolingMaxOp op) { InputAndOutputIndices indices = getInputAndOutputIndices(allIvs, op); // Emit scalar form. IndexedValueType output(op.output()); - IndexedValueType input(op.input()); Value lhs = output(indices.outputs); - Value rhs = input(indices.inputs); + Value rhs = getPoolingInput(op, indices.inputs); using edsc::op::sgt; Value maxValue = std_select(sgt(lhs, rhs), lhs, rhs); output(indices.outputs) = maxValue; @@ -402,7 +424,7 @@ IndexedValueType output(op.output()); IndexedValueType input(op.input()); Value lhs = output(indices.outputs); - Value rhs = input(indices.inputs); + Value rhs = getPoolingInput(op, indices.inputs); using edsc::op::slt; Value minValue = std_select(slt(lhs, rhs), lhs, rhs); output(indices.outputs) = minValue; @@ -410,10 +432,11 @@ template void emitScalarImplementation(ArrayRef allIvs, PoolingSumOp op) { auto indices = getInputAndOutputIndices(allIvs, op); - IndexedValueType input(op.input()), output(op.output()); + IndexedValueType output(op.output()); // Emit scalar form. - output(indices.outputs) += input(indices.inputs); + output(indices.outputs) += + getPoolingInput(op, indices.inputs); } /// Emits the MLIR for the scalar part of the indexed generic op by: /// 1. Emitting load ops for each input and output view in order. This is diff --git a/mlir/test/Dialect/Linalg/loops.mlir b/mlir/test/Dialect/Linalg/loops.mlir --- a/mlir/test/Dialect/Linalg/loops.mlir +++ b/mlir/test/Dialect/Linalg/loops.mlir @@ -17,6 +17,8 @@ // CHECKLOOP-DAG: #[[$convLowerBound:.*]] = affine_map<()[s0] -> (s0 floordiv 2)> // CHECKLOOP-DAG: #[[$convUpperBound:.*]] = affine_map<()[s0, s1] -> (s1 + s0 floordiv 2 - s0 + 1)> // CHECKLOOP-DAG: #[[$convMap:.*]] = affine_map<(d0, d1)[s0] -> (d0 + d1 - s0 floordiv 2)> +// CHECKLOOP-DAG: #[[$stride1Dilation1Padding1:.*]] = affine_map<(d0, d1) -> (d0 + d1 - 1)> +// CHECKLOOP-DAG: #[[$stride1Dilation1Padding2:.*]] = affine_map<(d0, d1) -> (d0 + d1 - 2)> // CHECKPARALLEL-DAG: #[[$strided1D:.*]] = affine_map<(d0)[s0] -> (d0 + s0)> // CHECKPARALLEL-DAG: #[[$strided2D:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)> @@ -31,6 +33,8 @@ // CHECKPARALLEL-DAG: #[[$convLowerBound:.*]] = affine_map<()[s0] -> (s0 floordiv 2)> // CHECKPARALLEL-DAG: #[[$convUpperBound:.*]] = affine_map<()[s0, s1] -> (s1 + s0 floordiv 2 - s0 + 1)> // CHECKPARALLEL-DAG: #[[$convMap:.*]] = affine_map<(d0, d1)[s0] -> (d0 + d1 - s0 floordiv 2)> +// CHECKPARALLEL-DAG: #[[$stride1Dilation1Padding1:.*]] = affine_map<(d0, d1) -> (d0 + d1 - 1)> +// CHECKPARALLEL-DAG: #[[$stride1Dilation1Padding2:.*]] = affine_map<(d0, d1) -> (d0 + d1 - 2)> func @matmul(%arg0: memref, %M: index, %N: index, %K: index) { @@ -470,6 +474,54 @@ // CHECKPARALLEL: %[[RES:.*]] = select %{{.*}}, %{{.*}}, %{{.*}} : f32 // CHECKPARALLEL: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref +func @pooling_max_padding(%arg0: memref, + %arg1: memref, + %arg2: memref) { + linalg.pooling_max(%arg0, %arg1, %arg2) { padding = dense<[[2, 2], [1, 1]]> : tensor<2x2xi64> } : + memref, memref, memref + return +} +// CHECKLOOP-LABEL: func @pooling_max_padding +// CHECKLOOP: %[[WX:.*]] = dim %arg1, %c0 : memref +// CHECKLOOP: %[[WY:.*]] = dim %arg1, %c1 : memref +// CHECKLOOP: %[[OX:.*]] = dim %arg2, %c0 : memref +// CHECKLOOP: %[[OY:.*]] = dim %arg2, %c1 : memref +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[OX]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[OY]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} { +// CHECKLOOP: %[[IX:.*]] = affine.apply #[[$stride1Dilation1Padding2]](%{{.*}}, %{{.*}}) +// CHECKLOOP: %[[IY:.*]] = affine.apply #[[$stride1Dilation1Padding1]](%{{.*}}, %{{.*}}) +// CHECKLOOP: %[[RHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref +// CHECKLOOP: %[[IDX:.*]] = affine.max #[[$clampMinMap]](%[[IX]]) +// CHECKLOOP: %[[IDY:.*]] = affine.max #[[$clampMinMap]](%[[IY]]) +// CHECKLOOP: %[[LHS:.*]] = load %{{.*}}[%[[IDX]], %[[IDY]]] : memref +// CHECKLOOP: %[[SEL:.*]] = select %{{.*}}, %{{.*}}, %[[LHS]] : f32 +// CHECKLOOP: %[[CMP:.*]] = cmpf "ogt", %[[RHS]], %[[SEL]] : f32 +// CHECKLOOP: %[[RES:.*]] = select %{{.*}}, %[[RHS]], %[[SEL]] : f32 +// CHECKLOOP: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref + +// CHECKPARALLEL-LABEL: func @pooling_max_padding +// CHECKPARALLEL: %[[WX:.*]] = dim %arg1, %c0 : memref +// CHECKPARALLEL: %[[WY:.*]] = dim %arg1, %c1 : memref +// CHECKPARALLEL: %[[OX:.*]] = dim %arg2, %c0 : memref +// CHECKPARALLEL: %[[OY:.*]] = dim %arg2, %c1 : memref +// CHECKPARALLEL: scf.parallel (%{{.*}}, %{{.*}}) = (%{{.*}}, %{{.*}}) to (%[[OX]], %[[OY]]) step (%{{.*}}, %{{.*}}) { +// CHECKPARALLEL: scf.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} { +// CHECKPARALLEL: scf.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} { +// CHECKPARALLEL: %[[IX:.*]] = affine.apply #[[$stride1Dilation1Padding2]](%{{.*}}, %{{.*}}) +// CHECKPARALLEL: %[[IY:.*]] = affine.apply #[[$stride1Dilation1Padding1]](%{{.*}}, %{{.*}}) +// CHECKPARALLEL: %[[RHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref +// CHECKPARALLEL: %[[IDX:.*]] = affine.max #[[$clampMinMap]](%[[IX]]) +// CHECKPARALLEL: %[[IDY:.*]] = affine.max #[[$clampMinMap]](%[[IY]]) +// CHECKPARALLEL: %[[LHS:.*]] = load %{{.*}}[%[[IDX]], %[[IDY]]] : memref +// CHECKPARALLEL: %[[SEL:.*]] = select %{{.*}}, %{{.*}}, %[[LHS]] : f32 +// CHECKPARALLEL: %[[CMP:.*]] = cmpf "ogt", %[[RHS]], %[[SEL]] : f32 +// CHECKPARALLEL: %[[RES:.*]] = select %{{.*}}, %[[RHS]], %[[SEL]] : f32 +// CHECKPARALLEL: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref + + + func @pooling_min(%arg0: memref, %arg1: memref, %arg2: memref) { @@ -508,6 +560,52 @@ // CHECKPARALLEL: %[[RES:.*]] = select %{{.*}}, %{{.*}}, %{{.*}} : f32 // CHECKPARALLEL: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref +func @pooling_min_padding(%arg0: memref, + %arg1: memref, + %arg2: memref) { + linalg.pooling_min(%arg0, %arg1, %arg2) { padding = dense<[[2, 2], [1, 1]]> : tensor<2x2xi64> } : + memref, memref, memref + return +} +// CHECKLOOP-LABEL: func @pooling_min_padding +// CHECKLOOP: %[[WX:.*]] = dim %arg1, %c0 : memref +// CHECKLOOP: %[[WY:.*]] = dim %arg1, %c1 : memref +// CHECKLOOP: %[[OX:.*]] = dim %arg2, %c0 : memref +// CHECKLOOP: %[[OY:.*]] = dim %arg2, %c1 : memref +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[OX]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[OY]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} { +// CHECKLOOP: %[[IX:.*]] = affine.apply #[[$stride1Dilation1Padding2]](%{{.*}}, %{{.*}}) +// CHECKLOOP: %[[IY:.*]] = affine.apply #[[$stride1Dilation1Padding1]](%{{.*}}, %{{.*}}) +// CHECKLOOP: %[[RHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref +// CHECKLOOP: %[[IDX:.*]] = affine.max #[[$clampMinMap]](%[[IX]]) +// CHECKLOOP: %[[IDY:.*]] = affine.max #[[$clampMinMap]](%[[IY]]) +// CHECKLOOP: %[[LHS:.*]] = load %{{.*}}[%[[IDX]], %[[IDY]]] : memref +// CHECKLOOP: %[[SEL:.*]] = select %{{.*}}, %{{.*}}, %[[LHS]] : f32 +// CHECKLOOP: %[[CMP:.*]] = cmpf "olt", %[[RHS]], %[[SEL]] : f32 +// CHECKLOOP: %[[RES:.*]] = select %{{.*}}, %[[RHS]], %[[SEL]] : f32 +// CHECKLOOP: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref + +// CHECKPARALLEL-LABEL: func @pooling_min_padding +// CHECKPARALLEL: %[[WX:.*]] = dim %arg1, %c0 : memref +// CHECKPARALLEL: %[[WY:.*]] = dim %arg1, %c1 : memref +// CHECKPARALLEL: %[[OX:.*]] = dim %arg2, %c0 : memref +// CHECKPARALLEL: %[[OY:.*]] = dim %arg2, %c1 : memref +// CHECKPARALLEL: scf.parallel (%{{.*}}, %{{.*}}) = (%{{.*}}, %{{.*}}) to (%[[OX]], %[[OY]]) step (%{{.*}}, %{{.*}}) { +// CHECKPARALLEL: scf.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} { +// CHECKPARALLEL: scf.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} { +// CHECKPARALLEL: %[[IX:.*]] = affine.apply #[[$stride1Dilation1Padding2]](%{{.*}}, %{{.*}}) +// CHECKPARALLEL: %[[IY:.*]] = affine.apply #[[$stride1Dilation1Padding1]](%{{.*}}, %{{.*}}) +// CHECKPARALLEL: %[[RHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref +// CHECKPARALLEL: %[[IDX:.*]] = affine.max #[[$clampMinMap]](%[[IX]]) +// CHECKPARALLEL: %[[IDY:.*]] = affine.max #[[$clampMinMap]](%[[IY]]) +// CHECKPARALLEL: %[[LHS:.*]] = load %{{.*}}[%[[IDX]], %[[IDY]]] : memref +// CHECKPARALLEL: %[[SEL:.*]] = select %{{.*}}, %{{.*}}, %[[LHS]] : f32 +// CHECKPARALLEL: %[[CMP:.*]] = cmpf "olt", %[[RHS]], %[[SEL]] : f32 +// CHECKPARALLEL: %[[RES:.*]] = select %{{.*}}, %[[RHS]], %[[SEL]] : f32 +// CHECKPARALLEL: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref + func @pooling_sum(%arg0: memref, %arg1: memref, %arg2: memref) { @@ -546,6 +644,52 @@ // CHECKPARALLEL: %[[RES:.*]] = addf %[[LHS]], %[[RHS]] : f32 // CHECKPARALLEL: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref +func @pooling_sum_padding(%arg0: memref, + %arg1: memref, + %arg2: memref) { + linalg.pooling_sum(%arg0, %arg1, %arg2) { padding = dense<[[2, 2], [1, 1]]> : tensor<2x2xi64> } : + memref, memref, memref + return +} + +// CHECKLOOP-LABEL: func @pooling_sum_padding +// CHECKLOOP: %[[WX:.*]] = dim %arg1, %c0 : memref +// CHECKLOOP: %[[WY:.*]] = dim %arg1, %c1 : memref +// CHECKLOOP: %[[OX:.*]] = dim %arg2, %c0 : memref +// CHECKLOOP: %[[OY:.*]] = dim %arg2, %c1 : memref +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[OX]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[OY]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} { +// CHECKLOOP: scf.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} { +// CHECKLOOP: %[[IX:.*]] = affine.apply #[[$stride1Dilation1Padding2]](%{{.*}}, %{{.*}}) +// CHECKLOOP: %[[IY:.*]] = affine.apply #[[$stride1Dilation1Padding1]](%{{.*}}, %{{.*}}) +// CHECKLOOP: %[[IDX:.*]] = affine.max #[[$clampMinMap]](%[[IX]]) +// CHECKLOOP: %[[IDY:.*]] = affine.max #[[$clampMinMap]](%[[IY]]) +// CHECKLOOP: %[[LHS:.*]] = load %{{.*}}[%[[IDX]], %[[IDY]]] : memref +// CHECKLOOP: %[[SEL:.*]] = select %{{.*}}, %{{.*}}, %[[LHS]] : f32 +// CHECKLOOP: %[[RHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref +// CHECKLOOP: %[[RES:.*]] = addf %[[RHS]], %[[SEL]] : f32 +// CHECKLOOP: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref + +// CHECKPARALLEL-LABEL: func @pooling_sum_padding +// CHECKPARALLEL: %[[WX:.*]] = dim %arg1, %c0 : memref +// CHECKPARALLEL: %[[WY:.*]] = dim %arg1, %c1 : memref +// CHECKPARALLEL: %[[OX:.*]] = dim %arg2, %c0 : memref +// CHECKPARALLEL: %[[OY:.*]] = dim %arg2, %c1 : memref +// CHECKPARALLEL: scf.parallel (%{{.*}}, %{{.*}}) = (%{{.*}}, %{{.*}}) to (%[[OX]], %[[OY]]) step (%{{.*}}, %{{.*}}) { +// CHECKPARALLEL: scf.for %{{.*}} = %{{.*}} to %[[WX]] step %{{.*}} { +// CHECKPARALLEL: scf.for %{{.*}} = %{{.*}} to %[[WY]] step %{{.*}} { +// CHECKPARALLEL: %[[IX:.*]] = affine.apply #[[$stride1Dilation1Padding2]](%{{.*}}, %{{.*}}) +// CHECKPARALLEL: %[[IY:.*]] = affine.apply #[[$stride1Dilation1Padding1]](%{{.*}}, %{{.*}}) +// CHECKPARALLEL: %[[IDX:.*]] = affine.max #[[$clampMinMap]](%[[IX]]) +// CHECKPARALLEL: %[[IDY:.*]] = affine.max #[[$clampMinMap]](%[[IY]]) +// CHECKPARALLEL: %[[LHS:.*]] = load %{{.*}}[%[[IDX]], %[[IDY]]] : memref +// CHECKPARALLEL: %[[SEL:.*]] = select %{{.*}}, %{{.*}}, %[[LHS]] : f32 +// CHECKPARALLEL: %[[RHS:.*]] = load %{{.*}}[%{{.*}}, %{{.*}}] : memref +// CHECKPARALLEL: %[[RES:.*]] = addf %[[RHS]], %[[SEL]] : f32 +// CHECKPARALLEL: store %[[RES]], %{{.*}}[%{{.*}}, %{{.*}}] : memref + + #accesses = [ affine_map<(i, j, k) -> (i, j)>, affine_map<(i, j, k) -> (i, j, k)>,