diff --git a/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h b/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h --- a/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h +++ b/mlir/include/mlir/Dialect/SparseTensor/Utils/Merger.h @@ -20,12 +20,26 @@ namespace mlir { namespace sparse_tensor { -/// Tensor expression kind. -enum class Kind { kTensor, kInvariant, kMulF, kMulI, kAddF, kAddI }; - /// Dimension level type for a tensor (undef means index does not appear). enum class Dim { kSparse, kDense, kSingle, kUndef }; +/// Tensor expression kind. +enum class Kind { + // Leaf. + kTensor, + kInvariant, + // Unary. + kNegF, + kNegI, + // Binary. + kMulF, + kMulI, + kAddF, + kAddI, + kSubF, + kSubI +}; + /// Tensor expression. Represents a MLIR expression in tensor index notation. /// For tensors, e0 denotes the tensor index. For invariants, the IR value is /// stored directly. For binary operations, e0 and e1 denote the index of the @@ -33,11 +47,12 @@ struct TensorExp { TensorExp(Kind k, unsigned x, unsigned y, Value v) : kind(k), e0(x), e1(y), val(v) { - assert((kind == Kind::kTensor && e0 != -1u && e1 == -1u && !val) || - (kind == Kind::kInvariant && e0 == -1u && e1 == -1u && val) || - (kind >= Kind::kMulF && e0 != -1u && e1 != -1u && !val)); + verify(); } + /// Verifies structure. + void verify(); + /// Tensor expression kind. Kind kind; @@ -111,6 +126,9 @@ /// Returns the index of the new set. unsigned takeDisj(Kind kind, unsigned s0, unsigned s1); + // Identity merge over a lattice set. Returns the index of the new set. + unsigned takeId(Kind kind, unsigned s); + /// Optimizes the iteration lattice points in the given set. This /// method should be called right before code generation to avoid /// generating redundant loops and conditions. diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -626,13 +626,27 @@ linalg::GenericOp op, unsigned exp) { if (merger.exp(exp).kind == Kind::kTensor) return genTensorLoad(merger, codegen, rewriter, op, exp); - else if (merger.exp(exp).kind == Kind::kInvariant) + if (merger.exp(exp).kind == Kind::kInvariant) return genInvariantValue(merger, codegen, rewriter, exp); + if (merger.exp(exp).kind == Kind::kNegF) { + Value v = genExp(merger, codegen, rewriter, op, merger.exp(exp).e1); + return rewriter.create(op.getLoc(), v); + } + if (merger.exp(exp).kind == Kind::kNegI) { + // TODO: no negi in standard? + Value v = genExp(merger, codegen, rewriter, op, merger.exp(exp).e1); + Type tp = v.getType(); + Value z = + rewriter.create(op.getLoc(), tp, rewriter.getZeroAttr(tp)); + return rewriter.create(op.getLoc(), z, v); + } Value v0 = genExp(merger, codegen, rewriter, op, merger.exp(exp).e0); Value v1 = genExp(merger, codegen, rewriter, op, merger.exp(exp).e1); switch (merger.exp(exp).kind) { case Kind::kTensor: case Kind::kInvariant: + case Kind::kNegF: + case Kind::kNegI: llvm_unreachable("handled above"); case Kind::kMulF: return rewriter.create(op.getLoc(), v0, v1); @@ -642,6 +656,10 @@ return rewriter.create(op.getLoc(), v0, v1); case Kind::kAddI: return rewriter.create(op.getLoc(), v0, v1); + case Kind::kSubF: + return rewriter.create(op.getLoc(), v0, v1); + case Kind::kSubI: + return rewriter.create(op.getLoc(), v0, v1); } llvm_unreachable("unexpected expression kind"); } @@ -650,6 +668,8 @@ static void genInvariants(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter, linalg::GenericOp op, unsigned exp, unsigned ldx, bool hoist) { + if (exp == -1u) + return; if (merger.exp(exp).kind == Kind::kTensor) { // Inspect tensor indices. bool atLevel = ldx == -1u; diff --git a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp --- a/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp +++ b/mlir/lib/Dialect/SparseTensor/Utils/Merger.cpp @@ -14,6 +14,24 @@ namespace mlir { namespace sparse_tensor { +void TensorExp::verify() { + switch (kind) { + case Kind::kTensor: + assert(e0 != -1u && e1 == -1u && !val); + break; + case Kind::kInvariant: + assert(e0 == -1u && e1 == -1u && val); + break; + case Kind::kNegF: + case Kind::kNegI: + assert(e0 == -1u && e1 != -1u && !val); + break; + default: + assert(e0 != -1u && e1 != -1u && !val); + break; + } +} + // // Lattice methods. // @@ -48,6 +66,7 @@ unsigned Merger::takeConj(Kind kind, unsigned s0, unsigned s1) { unsigned s = addSet(); + // Take the conjuction for the full cross-product. for (unsigned p0 : latSets[s0]) for (unsigned p1 : latSets[s1]) latSets[s].push_back(conjLatPoint(kind, p0, p1)); @@ -55,14 +74,32 @@ } unsigned Merger::takeDisj(Kind kind, unsigned s0, unsigned s1) { + // Take the conjuction. unsigned s = takeConj(kind, s0, s1); + // Followed by all in s0. for (unsigned p : latSets[s0]) latSets[s].push_back(p); + // Break out unary for subtraction. + if (kind == Kind::kSubF) + s1 = takeId(Kind::kNegF, s1); + else if (kind == Kind::kSubI) + s1 = takeId(Kind::kNegI, s1); + // Followed by all in s1. for (unsigned p : latSets[s1]) latSets[s].push_back(p); return s; } +unsigned Merger::takeId(Kind kind, unsigned s0) { + unsigned s = addSet(); + for (unsigned p : latSets[s0]) { + unsigned e = addExp(kind, -1u, latPoints[p].exp); + latPoints.push_back(LatPoint(latPoints[p].bits, e)); + latSets[s].push_back(latPoints.size() - 1); + } + return s; +} + unsigned Merger::optimizeSet(unsigned s0) { unsigned s = addSet(); assert(latSets[s0].size() != 0); @@ -147,6 +184,26 @@ // Print methods (for debugging). // +static char kindToChar(Kind kind) { + switch (kind) { + case Kind::kTensor: + case Kind::kInvariant: + case Kind::kNegF: + case Kind::kNegI: + return '?'; + case Kind::kMulF: + case Kind::kMulI: + return '*'; + case Kind::kAddF: + case Kind::kAddI: + return '+'; + case Kind::kSubF: + case Kind::kSubI: + return '-'; + } + llvm_unreachable("unexpected kind"); +} + void Merger::dumpExp(unsigned e) const { switch (tensorExps[e].kind) { case Kind::kTensor: @@ -159,22 +216,17 @@ case Kind::kInvariant: llvm::dbgs() << "invariant"; break; - default: - case Kind::kMulI: - llvm::dbgs() << "("; - dumpExp(tensorExps[e].e0); - llvm::dbgs() << " * "; + case Kind::kNegF: + case Kind::kNegI: + llvm::dbgs() << "-"; dumpExp(tensorExps[e].e1); - llvm::dbgs() << ")"; break; - case Kind::kAddF: - case Kind::kAddI: + default: llvm::dbgs() << "("; dumpExp(tensorExps[e].e0); - llvm::dbgs() << " + "; + llvm::dbgs() << " " << kindToChar(tensorExps[e].kind) << " "; dumpExp(tensorExps[e].e1); llvm::dbgs() << ")"; - break; } } @@ -183,7 +235,7 @@ dumpBits(latPoints[p].bits); llvm::dbgs() << " :"; dumpBits(latPoints[p].simple); - llvm::dbgs() << " / "; + llvm::dbgs() << " : "; dumpExp(latPoints[p].exp); llvm::dbgs() << " )\n"; } @@ -229,7 +281,9 @@ unsigned Merger::buildLattices(unsigned e, unsigned idx) { Kind kind = tensorExps[e].kind; - if (kind == Kind::kTensor || kind == Kind::kInvariant) { + switch (kind) { + case Kind::kTensor: + case Kind::kInvariant: { // Either the index is really used in the tensor expression, or it is // set to the undefined index in that dimension. An invariant expression // is set to a synthetic tensor with undefined indices only. @@ -238,18 +292,22 @@ latSets[s].push_back(addLat(t, idx, e)); return s; } - unsigned s0 = buildLattices(tensorExps[e].e0, idx); - unsigned s1 = buildLattices(tensorExps[e].e1, idx); - switch (kind) { - case Kind::kTensor: - case Kind::kInvariant: - llvm_unreachable("handled above"); + case Kind::kNegF: + case Kind::kNegI: + return takeId(kind, // unary ID + buildLattices(tensorExps[e].e1, idx)); case Kind::kMulF: case Kind::kMulI: - return takeConj(kind, s0, s1); + return takeConj(kind, // binary conjunction + buildLattices(tensorExps[e].e0, idx), + buildLattices(tensorExps[e].e1, idx)); case Kind::kAddF: case Kind::kAddI: - return takeDisj(kind, s0, s1); + case Kind::kSubF: + case Kind::kSubI: + return takeDisj(kind, // binary disjunction + buildLattices(tensorExps[e].e0, idx), + buildLattices(tensorExps[e].e1, idx)); } llvm_unreachable("unexpected expression kind"); } @@ -279,6 +337,16 @@ Operation *def = val.getDefiningOp(); if (def->getBlock() != &op.region().front()) return addExp(Kind::kInvariant, val); + // Construct unary operations if subexpressions could be built. + if (def->getNumOperands() == 1) { + auto x = buildTensorExp(op, def->getOperand(0)); + if (x.hasValue()) { + unsigned e = x.getValue(); + if (isa(def)) + return addExp(Kind::kNegF, -1u, e); + // TODO: no negi in standard? + } + } // Construct binary operations if subexpressions could be built. if (def->getNumOperands() == 2) { auto x = buildTensorExp(op, def->getOperand(0)); @@ -294,6 +362,10 @@ return addExp(Kind::kAddF, e0, e1); if (isa(def)) return addExp(Kind::kAddI, e0, e1); + if (isa(def)) + return addExp(Kind::kSubF, e0, e1); + if (isa(def)) + return addExp(Kind::kSubI, e0, e1); } } // Cannot build. diff --git a/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir b/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Dialect/SparseTensor/sparse_fp_ops.mlir @@ -0,0 +1,213 @@ +// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py +// RUN: mlir-opt %s -sparsification | FileCheck %s + +#SV = #sparse_tensor.encoding<{ dimLevelType = [ "compressed" ] }> + +#trait1 = { + indexing_maps = [ + affine_map<(i) -> (i)>, // a + affine_map<(i) -> (i)> // x (out) + ], + iterator_types = ["parallel"], + doc = "x(i) = OP a(i)" +} + +#trait2 = { + indexing_maps = [ + affine_map<(i) -> (i)>, // a + affine_map<(i) -> (i)>, // b + affine_map<(i) -> (i)> // x (out) + ], + iterator_types = ["parallel"], + doc = "x(i) = a(i) OP b(i)" +} + +// CHECK-LABEL: func @neg( +// CHECK-SAME: %[[VAL_0:.*0]]: tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*1]]: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { +// CHECK: %[[VAL_2:.*]] = constant 0 : index +// CHECK: %[[VAL_3:.*]] = constant 1 : index +// CHECK: %[[VAL_4:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_2]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_5:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_2]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_6:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_7:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xf64> +// CHECK: %[[VAL_8:.*]] = memref.load %[[VAL_4]]{{\[}}%[[VAL_2]]] : memref +// CHECK: %[[VAL_9:.*]] = memref.load %[[VAL_4]]{{\[}}%[[VAL_3]]] : memref +// CHECK: scf.for %[[VAL_10:.*]] = %[[VAL_8]] to %[[VAL_9]] step %[[VAL_3]] { +// CHECK: %[[VAL_11:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_10]]] : memref +// CHECK: %[[VAL_12:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_10]]] : memref +// CHECK: %[[VAL_13:.*]] = negf %[[VAL_12]] : f64 +// CHECK: memref.store %[[VAL_13]], %[[VAL_7]]{{\[}}%[[VAL_11]]] : memref<32xf64> +// CHECK: } +// CHECK: %[[VAL_14:.*]] = memref.tensor_load %[[VAL_7]] : memref<32xf64> +// CHECK: return %[[VAL_14]] : tensor<32xf64> +// CHECK: } +func @neg(%arga: tensor<32xf64, #SV>, + %argx: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { + %0 = linalg.generic #trait1 + ins(%arga: tensor<32xf64, #SV>) + outs(%argx: tensor<32xf64>) { + ^bb(%a: f64, %x: f64): + %0 = negf %a : f64 + linalg.yield %0 : f64 + } -> tensor<32xf64> + return %0 : tensor<32xf64> +} + +// CHECK-LABEL: func @add( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<32xf64>, +// CHECK-SAME: %[[VAL_2:.*]]: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { +// CHECK: %[[VAL_3:.*]] = constant 32 : index +// CHECK: %[[VAL_4:.*]] = constant 0 : index +// CHECK: %[[VAL_5:.*]] = constant true +// CHECK: %[[VAL_6:.*]] = constant 1 : index +// CHECK: %[[VAL_7:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_4]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_8:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_4]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_9:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_10:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xf64> +// CHECK: %[[VAL_11:.*]] = memref.buffer_cast %[[VAL_2]] : memref<32xf64> +// CHECK: %[[VAL_12:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_4]]] : memref +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_6]]] : memref +// CHECK: %[[VAL_14:.*]]:2 = scf.while (%[[VAL_15:.*]] = %[[VAL_12]], %[[VAL_16:.*]] = %[[VAL_4]]) : (index, index) -> (index, index) { +// CHECK: %[[VAL_17:.*]] = cmpi ult, %[[VAL_15]], %[[VAL_13]] : index +// CHECK: scf.condition(%[[VAL_17]]) %[[VAL_15]], %[[VAL_16]] : index, index +// CHECK: } do { +// CHECK: ^bb0(%[[VAL_18:.*]]: index, %[[VAL_19:.*]]: index): +// CHECK: %[[VAL_20:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_18]]] : memref +// CHECK: %[[VAL_21:.*]] = cmpi eq, %[[VAL_20]], %[[VAL_19]] : index +// CHECK: scf.if %[[VAL_21]] { +// CHECK: %[[VAL_22:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_18]]] : memref +// CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: %[[VAL_24:.*]] = addf %[[VAL_22]], %[[VAL_23]] : f64 +// CHECK: memref.store %[[VAL_24]], %[[VAL_11]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: } else { +// CHECK: scf.if %[[VAL_5]] { +// CHECK: %[[VAL_25:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: memref.store %[[VAL_25]], %[[VAL_11]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: } else { +// CHECK: } +// CHECK: } +// CHECK: %[[VAL_26:.*]] = cmpi eq, %[[VAL_20]], %[[VAL_19]] : index +// CHECK: %[[VAL_27:.*]] = addi %[[VAL_18]], %[[VAL_6]] : index +// CHECK: %[[VAL_28:.*]] = select %[[VAL_26]], %[[VAL_27]], %[[VAL_18]] : index +// CHECK: %[[VAL_29:.*]] = addi %[[VAL_19]], %[[VAL_6]] : index +// CHECK: scf.yield %[[VAL_28]], %[[VAL_29]] : index, index +// CHECK: } +// CHECK: scf.for %[[VAL_30:.*]] = %[[VAL_31:.*]]#1 to %[[VAL_3]] step %[[VAL_6]] { +// CHECK: %[[VAL_32:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_30]]] : memref<32xf64> +// CHECK: memref.store %[[VAL_32]], %[[VAL_11]]{{\[}}%[[VAL_30]]] : memref<32xf64> +// CHECK: } +// CHECK: %[[VAL_33:.*]] = memref.tensor_load %[[VAL_11]] : memref<32xf64> +// CHECK: return %[[VAL_33]] : tensor<32xf64> +// CHECK: } +func @add(%arga: tensor<32xf64, #SV>, + %argb: tensor<32xf64>, + %argx: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { + %0 = linalg.generic #trait2 + ins(%arga, %argb: tensor<32xf64, #SV>, tensor<32xf64>) + outs(%argx: tensor<32xf64>) { + ^bb(%a: f64, %b: f64, %x: f64): + %0 = addf %a, %b : f64 + linalg.yield %0 : f64 + } -> tensor<32xf64> + return %0 : tensor<32xf64> +} + +// CHECK-LABEL: func @sub( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<32xf64>, +// CHECK-SAME: %[[VAL_2:.*]]: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { +// CHECK: %[[VAL_3:.*]] = constant 32 : index +// CHECK: %[[VAL_4:.*]] = constant 0 : index +// CHECK: %[[VAL_5:.*]] = constant true +// CHECK: %[[VAL_6:.*]] = constant 1 : index +// CHECK: %[[VAL_7:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_4]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_8:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_4]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_9:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_10:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xf64> +// CHECK: %[[VAL_11:.*]] = memref.buffer_cast %[[VAL_2]] : memref<32xf64> +// CHECK: %[[VAL_12:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_4]]] : memref +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_6]]] : memref +// CHECK: %[[VAL_14:.*]]:2 = scf.while (%[[VAL_15:.*]] = %[[VAL_12]], %[[VAL_16:.*]] = %[[VAL_4]]) : (index, index) -> (index, index) { +// CHECK: %[[VAL_17:.*]] = cmpi ult, %[[VAL_15]], %[[VAL_13]] : index +// CHECK: scf.condition(%[[VAL_17]]) %[[VAL_15]], %[[VAL_16]] : index, index +// CHECK: } do { +// CHECK: ^bb0(%[[VAL_18:.*]]: index, %[[VAL_19:.*]]: index): +// CHECK: %[[VAL_20:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_18]]] : memref +// CHECK: %[[VAL_21:.*]] = cmpi eq, %[[VAL_20]], %[[VAL_19]] : index +// CHECK: scf.if %[[VAL_21]] { +// CHECK: %[[VAL_22:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_18]]] : memref +// CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: %[[VAL_24:.*]] = subf %[[VAL_22]], %[[VAL_23]] : f64 +// CHECK: memref.store %[[VAL_24]], %[[VAL_11]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: } else { +// CHECK: scf.if %[[VAL_5]] { +// CHECK: %[[VAL_25:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: %[[VAL_26:.*]] = negf %[[VAL_25]] : f64 +// CHECK: memref.store %[[VAL_26]], %[[VAL_11]]{{\[}}%[[VAL_19]]] : memref<32xf64> +// CHECK: } else { +// CHECK: } +// CHECK: } +// CHECK: %[[VAL_27:.*]] = cmpi eq, %[[VAL_20]], %[[VAL_19]] : index +// CHECK: %[[VAL_28:.*]] = addi %[[VAL_18]], %[[VAL_6]] : index +// CHECK: %[[VAL_29:.*]] = select %[[VAL_27]], %[[VAL_28]], %[[VAL_18]] : index +// CHECK: %[[VAL_30:.*]] = addi %[[VAL_19]], %[[VAL_6]] : index +// CHECK: scf.yield %[[VAL_29]], %[[VAL_30]] : index, index +// CHECK: } +// CHECK: scf.for %[[VAL_31:.*]] = %[[VAL_32:.*]]#1 to %[[VAL_3]] step %[[VAL_6]] { +// CHECK: %[[VAL_33:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_31]]] : memref<32xf64> +// CHECK: %[[VAL_34:.*]] = negf %[[VAL_33]] : f64 +// CHECK: memref.store %[[VAL_34]], %[[VAL_11]]{{\[}}%[[VAL_31]]] : memref<32xf64> +// CHECK: } +// CHECK: %[[VAL_35:.*]] = memref.tensor_load %[[VAL_11]] : memref<32xf64> +// CHECK: return %[[VAL_35]] : tensor<32xf64> +// CHECK: } +func @sub(%arga: tensor<32xf64, #SV>, + %argb: tensor<32xf64>, + %argx: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { + %0 = linalg.generic #trait2 + ins(%arga, %argb: tensor<32xf64, #SV>, tensor<32xf64>) + outs(%argx: tensor<32xf64>) { + ^bb(%a: f64, %b: f64, %x: f64): + %0 = subf %a, %b : f64 + linalg.yield %0 : f64 + } -> tensor<32xf64> + return %0 : tensor<32xf64> +} + +// CHECK-LABEL: func @mul( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<32xf64>, +// CHECK-SAME: %[[VAL_2:.*]]: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { +// CHECK: %[[VAL_3:.*]] = constant 0 : index +// CHECK: %[[VAL_4:.*]] = constant 1 : index +// CHECK: %[[VAL_5:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_3]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_6:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_3]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_7:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xf64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_8:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xf64> +// CHECK: %[[VAL_9:.*]] = memref.buffer_cast %[[VAL_2]] : memref<32xf64> +// CHECK: %[[VAL_10:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_3]]] : memref +// CHECK: %[[VAL_11:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_4]]] : memref +// CHECK: scf.for %[[VAL_12:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_4]] { +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_12]]] : memref +// CHECK: %[[VAL_14:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_12]]] : memref +// CHECK: %[[VAL_15:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_13]]] : memref<32xf64> +// CHECK: %[[VAL_16:.*]] = mulf %[[VAL_14]], %[[VAL_15]] : f64 +// CHECK: memref.store %[[VAL_16]], %[[VAL_9]]{{\[}}%[[VAL_13]]] : memref<32xf64> +// CHECK: } +// CHECK: %[[VAL_17:.*]] = memref.tensor_load %[[VAL_9]] : memref<32xf64> +// CHECK: return %[[VAL_17]] : tensor<32xf64> +// CHECK: } +func @mul(%arga: tensor<32xf64, #SV>, + %argb: tensor<32xf64>, + %argx: tensor<32xf64> {linalg.inplaceable = true}) -> tensor<32xf64> { + %0 = linalg.generic #trait2 + ins(%arga, %argb: tensor<32xf64, #SV>, tensor<32xf64>) + outs(%argx: tensor<32xf64>) { + ^bb(%a: f64, %b: f64, %x: f64): + %0 = mulf %a, %b : f64 + linalg.yield %0 : f64 + } -> tensor<32xf64> + return %0 : tensor<32xf64> +} diff --git a/mlir/test/Dialect/SparseTensor/sparse_int_ops.mlir b/mlir/test/Dialect/SparseTensor/sparse_int_ops.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Dialect/SparseTensor/sparse_int_ops.mlir @@ -0,0 +1,173 @@ +// NOTE: Assertions have been autogenerated by utils/generate-test-checks.py +// RUN: mlir-opt %s -sparsification | FileCheck %s + +#SV = #sparse_tensor.encoding<{ dimLevelType = [ "compressed" ] }> + +#trait2 = { + indexing_maps = [ + affine_map<(i) -> (i)>, // a + affine_map<(i) -> (i)>, // b + affine_map<(i) -> (i)> // x (out) + ], + iterator_types = ["parallel"], + doc = "x(i) = a(i) OP b(i)" +} + +// CHECK-LABEL: func @add( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<32xi64>, +// CHECK-SAME: %[[VAL_2:.*]]: tensor<32xi64> {linalg.inplaceable = true}) -> tensor<32xi64> { +// CHECK: %[[VAL_3:.*]] = constant 32 : index +// CHECK: %[[VAL_4:.*]] = constant 0 : index +// CHECK: %[[VAL_5:.*]] = constant true +// CHECK: %[[VAL_6:.*]] = constant 1 : index +// CHECK: %[[VAL_7:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_4]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_8:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_4]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_9:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_10:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xi64> +// CHECK: %[[VAL_11:.*]] = memref.buffer_cast %[[VAL_2]] : memref<32xi64> +// CHECK: %[[VAL_12:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_4]]] : memref +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_6]]] : memref +// CHECK: %[[VAL_14:.*]]:2 = scf.while (%[[VAL_15:.*]] = %[[VAL_12]], %[[VAL_16:.*]] = %[[VAL_4]]) : (index, index) -> (index, index) { +// CHECK: %[[VAL_17:.*]] = cmpi ult, %[[VAL_15]], %[[VAL_13]] : index +// CHECK: scf.condition(%[[VAL_17]]) %[[VAL_15]], %[[VAL_16]] : index, index +// CHECK: } do { +// CHECK: ^bb0(%[[VAL_18:.*]]: index, %[[VAL_19:.*]]: index): +// CHECK: %[[VAL_20:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_18]]] : memref +// CHECK: %[[VAL_21:.*]] = cmpi eq, %[[VAL_20]], %[[VAL_19]] : index +// CHECK: scf.if %[[VAL_21]] { +// CHECK: %[[VAL_22:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_18]]] : memref +// CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref<32xi64> +// CHECK: %[[VAL_24:.*]] = addi %[[VAL_22]], %[[VAL_23]] : i64 +// CHECK: memref.store %[[VAL_24]], %[[VAL_11]]{{\[}}%[[VAL_19]]] : memref<32xi64> +// CHECK: } else { +// CHECK: scf.if %[[VAL_5]] { +// CHECK: %[[VAL_25:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref<32xi64> +// CHECK: memref.store %[[VAL_25]], %[[VAL_11]]{{\[}}%[[VAL_19]]] : memref<32xi64> +// CHECK: } else { +// CHECK: } +// CHECK: } +// CHECK: %[[VAL_26:.*]] = cmpi eq, %[[VAL_20]], %[[VAL_19]] : index +// CHECK: %[[VAL_27:.*]] = addi %[[VAL_18]], %[[VAL_6]] : index +// CHECK: %[[VAL_28:.*]] = select %[[VAL_26]], %[[VAL_27]], %[[VAL_18]] : index +// CHECK: %[[VAL_29:.*]] = addi %[[VAL_19]], %[[VAL_6]] : index +// CHECK: scf.yield %[[VAL_28]], %[[VAL_29]] : index, index +// CHECK: } +// CHECK: scf.for %[[VAL_30:.*]] = %[[VAL_31:.*]]#1 to %[[VAL_3]] step %[[VAL_6]] { +// CHECK: %[[VAL_32:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_30]]] : memref<32xi64> +// CHECK: memref.store %[[VAL_32]], %[[VAL_11]]{{\[}}%[[VAL_30]]] : memref<32xi64> +// CHECK: } +// CHECK: %[[VAL_33:.*]] = memref.tensor_load %[[VAL_11]] : memref<32xi64> +// CHECK: return %[[VAL_33]] : tensor<32xi64> +// CHECK: } +func @add(%arga: tensor<32xi64, #SV>, + %argb: tensor<32xi64>, + %argx: tensor<32xi64> {linalg.inplaceable = true}) -> tensor<32xi64> { + %0 = linalg.generic #trait2 + ins(%arga, %argb: tensor<32xi64, #SV>, tensor<32xi64>) + outs(%argx: tensor<32xi64>) { + ^bb(%a: i64, %b: i64, %x: i64): + %0 = addi %a, %b : i64 + linalg.yield %0 : i64 + } -> tensor<32xi64> + return %0 : tensor<32xi64> +} + +// CHECK-LABEL: func @sub( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<32xi64>, +// CHECK-SAME: %[[VAL_2:.*]]: tensor<32xi64> {linalg.inplaceable = true}) -> tensor<32xi64> { +// CHECK: %[[VAL_3:.*]] = constant 32 : index +// CHECK: %[[VAL_4:.*]] = constant 0 : index +// CHECK: %[[VAL_5:.*]] = constant true +// CHECK: %[[VAL_6:.*]] = constant 1 : index +// CHECK: %[[VAL_7:.*]] = constant 0 : i64 +// CHECK: %[[VAL_8:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_4]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_9:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_4]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_10:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_11:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xi64> +// CHECK: %[[VAL_12:.*]] = memref.buffer_cast %[[VAL_2]] : memref<32xi64> +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_4]]] : memref +// CHECK: %[[VAL_14:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_6]]] : memref +// CHECK: %[[VAL_15:.*]]:2 = scf.while (%[[VAL_16:.*]] = %[[VAL_13]], %[[VAL_17:.*]] = %[[VAL_4]]) : (index, index) -> (index, index) { +// CHECK: %[[VAL_18:.*]] = cmpi ult, %[[VAL_16]], %[[VAL_14]] : index +// CHECK: scf.condition(%[[VAL_18]]) %[[VAL_16]], %[[VAL_17]] : index, index +// CHECK: } do { +// CHECK: ^bb0(%[[VAL_19:.*]]: index, %[[VAL_20:.*]]: index): +// CHECK: %[[VAL_21:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_19]]] : memref +// CHECK: %[[VAL_22:.*]] = cmpi eq, %[[VAL_21]], %[[VAL_20]] : index +// CHECK: scf.if %[[VAL_22]] { +// CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_19]]] : memref +// CHECK: %[[VAL_24:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_20]]] : memref<32xi64> +// CHECK: %[[VAL_25:.*]] = subi %[[VAL_23]], %[[VAL_24]] : i64 +// CHECK: memref.store %[[VAL_25]], %[[VAL_12]]{{\[}}%[[VAL_20]]] : memref<32xi64> +// CHECK: } else { +// CHECK: scf.if %[[VAL_5]] { +// CHECK: %[[VAL_26:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_20]]] : memref<32xi64> +// CHECK: %[[VAL_27:.*]] = subi %[[VAL_7]], %[[VAL_26]] : i64 +// CHECK: memref.store %[[VAL_27]], %[[VAL_12]]{{\[}}%[[VAL_20]]] : memref<32xi64> +// CHECK: } else { +// CHECK: } +// CHECK: } +// CHECK: %[[VAL_28:.*]] = cmpi eq, %[[VAL_21]], %[[VAL_20]] : index +// CHECK: %[[VAL_29:.*]] = addi %[[VAL_19]], %[[VAL_6]] : index +// CHECK: %[[VAL_30:.*]] = select %[[VAL_28]], %[[VAL_29]], %[[VAL_19]] : index +// CHECK: %[[VAL_31:.*]] = addi %[[VAL_20]], %[[VAL_6]] : index +// CHECK: scf.yield %[[VAL_30]], %[[VAL_31]] : index, index +// CHECK: } +// CHECK: scf.for %[[VAL_32:.*]] = %[[VAL_33:.*]]#1 to %[[VAL_3]] step %[[VAL_6]] { +// CHECK: %[[VAL_34:.*]] = memref.load %[[VAL_11]]{{\[}}%[[VAL_32]]] : memref<32xi64> +// CHECK: %[[VAL_35:.*]] = subi %[[VAL_7]], %[[VAL_34]] : i64 +// CHECK: memref.store %[[VAL_35]], %[[VAL_12]]{{\[}}%[[VAL_32]]] : memref<32xi64> +// CHECK: } +// CHECK: %[[VAL_36:.*]] = memref.tensor_load %[[VAL_12]] : memref<32xi64> +// CHECK: return %[[VAL_36]] : tensor<32xi64> +// CHECK: } +func @sub(%arga: tensor<32xi64, #SV>, + %argb: tensor<32xi64>, + %argx: tensor<32xi64> {linalg.inplaceable = true}) -> tensor<32xi64> { + %0 = linalg.generic #trait2 + ins(%arga, %argb: tensor<32xi64, #SV>, tensor<32xi64>) + outs(%argx: tensor<32xi64>) { + ^bb(%a: i64, %b: i64, %x: i64): + %0 = subi %a, %b : i64 + linalg.yield %0 : i64 + } -> tensor<32xi64> + return %0 : tensor<32xi64> +} + +// CHECK-LABEL: func @mul( +// CHECK-SAME: %[[VAL_0:.*]]: tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<32xi64>, +// CHECK-SAME: %[[VAL_2:.*]]: tensor<32xi64> {linalg.inplaceable = true}) -> tensor<32xi64> { +// CHECK: %[[VAL_3:.*]] = constant 0 : index +// CHECK: %[[VAL_4:.*]] = constant 1 : index +// CHECK: %[[VAL_5:.*]] = sparse_tensor.pointers %[[VAL_0]], %[[VAL_3]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_6:.*]] = sparse_tensor.indices %[[VAL_0]], %[[VAL_3]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_7:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<32xi64, #sparse_tensor.encoding<{{{.*}}}>> +// CHECK: %[[VAL_8:.*]] = memref.buffer_cast %[[VAL_1]] : memref<32xi64> +// CHECK: %[[VAL_9:.*]] = memref.buffer_cast %[[VAL_2]] : memref<32xi64> +// CHECK: %[[VAL_10:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_3]]] : memref +// CHECK: %[[VAL_11:.*]] = memref.load %[[VAL_5]]{{\[}}%[[VAL_4]]] : memref +// CHECK: scf.for %[[VAL_12:.*]] = %[[VAL_10]] to %[[VAL_11]] step %[[VAL_4]] { +// CHECK: %[[VAL_13:.*]] = memref.load %[[VAL_6]]{{\[}}%[[VAL_12]]] : memref +// CHECK: %[[VAL_14:.*]] = memref.load %[[VAL_7]]{{\[}}%[[VAL_12]]] : memref +// CHECK: %[[VAL_15:.*]] = memref.load %[[VAL_8]]{{\[}}%[[VAL_13]]] : memref<32xi64> +// CHECK: %[[VAL_16:.*]] = muli %[[VAL_14]], %[[VAL_15]] : i64 +// CHECK: memref.store %[[VAL_16]], %[[VAL_9]]{{\[}}%[[VAL_13]]] : memref<32xi64> +// CHECK: } +// CHECK: %[[VAL_17:.*]] = memref.tensor_load %[[VAL_9]] : memref<32xi64> +// CHECK: return %[[VAL_17]] : tensor<32xi64> +// CHECK: } +func @mul(%arga: tensor<32xi64, #SV>, + %argb: tensor<32xi64>, + %argx: tensor<32xi64> {linalg.inplaceable = true}) -> tensor<32xi64> { + %0 = linalg.generic #trait2 + ins(%arga, %argb: tensor<32xi64, #SV>, tensor<32xi64>) + outs(%argx: tensor<32xi64>) { + ^bb(%a: i64, %b: i64, %x: i64): + %0 = muli %a, %b : i64 + linalg.yield %0 : i64 + } -> tensor<32xi64> + return %0 : tensor<32xi64> +}