diff --git a/mlir/include/mlir/Dialect/X86Vector/Transforms.h b/mlir/include/mlir/Dialect/X86Vector/Transforms.h --- a/mlir/include/mlir/Dialect/X86Vector/Transforms.h +++ b/mlir/include/mlir/Dialect/X86Vector/Transforms.h @@ -23,6 +23,28 @@ /// Helper class to factor out the creation and extraction of masks from nibs. struct MaskHelper { + /// b0 captures the lowest bit, b7 captures the highest bit. + /// Meant to be used with instructions such as mm256BlendPs. + template + static char blend() { + static_assert(b0 <= 1 && b1 <= 1 && b2 <= 1 && b3 <= 1, "overflow"); + static_assert(b4 <= 1 && b5 <= 1 && b6 <= 1 && b7 <= 1, "overflow"); + return (b7 << 7) + (b6 << 6) + (b5 << 5) + (b4 << 4) + (b3 << 3) + + (b2 << 2) + (b1 << 1) + b0; + } + /// b0 captures the lowest bit, b7 captures the highest bit. + /// Meant to be used with instructions such as mm256BlendPs. + static void extractBlend(char mask, char &b0, char &b1, char &b2, char &b3, + char &b4, char &b5, char &b6, char &b7) { + b7 = mask & (1 << 7); + b6 = mask & (1 << 6); + b5 = mask & (1 << 5); + b4 = mask & (1 << 4); + b3 = mask & (1 << 3); + b2 = mask & (1 << 2); + b1 = mask & (1 << 1); + b0 = mask & 1; + } /// b01 captures the lower 2 bits, b67 captures the higher 2 bits. /// Meant to be used with instructions such as mm256ShufflePs. template @@ -70,6 +92,18 @@ namespace avx2 { +namespace inline_asm { +//===----------------------------------------------------------------------===// +/// Methods in the inline_asmasm namespace emit calls to LLVM::InlineAsmOp. +//===----------------------------------------------------------------------===// +/// If bit i of `mask` is zero, take f32@i from v1 else take it from v2. +Value mm256BlendPs(ImplicitLocOpBuilder &b, Value v1, Value v2, char mask); +} // namespace inline_asm + +namespace intrin { +//===----------------------------------------------------------------------===// +/// Methods in the intrin namespace emulate clang's impl. of X86 immintrin. +//===----------------------------------------------------------------------===// /// Lower to vector.shuffle v1, v2, [0, 8, 1, 9, 4, 12, 5, 13]. Value mm256UnpackLoPs(ImplicitLocOpBuilder &b, Value v1, Value v2); @@ -91,6 +125,13 @@ Value mm256Permute2f128Ps(ImplicitLocOpBuilder &b, Value v1, Value v2, int8_t mask); +/// If bit i of `mask` is zero, take f32@i from v1 else take it from v2. +Value mm256BlendPs(ImplicitLocOpBuilder &b, Value v1, Value v2, char mask); +} // namespace intrin + +//===----------------------------------------------------------------------===// +/// Generic lowerings may either use intrin or inline_asm depending on needs. +//===----------------------------------------------------------------------===// /// 4x8xf32-specific AVX2 transpose lowering. void transpose4x8xf32(ImplicitLocOpBuilder &ib, MutableArrayRef vs); diff --git a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp --- a/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp +++ b/mlir/lib/Dialect/X86Vector/Transforms/AVXTranspose.cpp @@ -11,25 +11,48 @@ // //===----------------------------------------------------------------------===// +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/Vector/VectorOps.h" #include "mlir/Dialect/X86Vector/Transforms.h" #include "mlir/IR/ImplicitLocOpBuilder.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/FormatVariadic.h" using namespace mlir; using namespace mlir::vector; using namespace mlir::x86vector; using namespace mlir::x86vector::avx2; +using namespace mlir::x86vector::avx2::inline_asm; +using namespace mlir::x86vector::avx2::intrin; -Value mlir::x86vector::avx2::mm256UnpackLoPs(ImplicitLocOpBuilder &b, Value v1, - Value v2) { +Value mlir::x86vector::avx2::inline_asm::mm256BlendPs(ImplicitLocOpBuilder &b, + Value v1, Value v2, + char mask) { + auto asmDialectAttr = + LLVM::AsmDialectAttr::get(b.getContext(), LLVM::AsmDialect::AD_Intel); + + // clang-format off + auto asmTp = "vblendps $0, $1, $2, {0}"; + auto asmCstr = "=x,x,x"; // Careful: constraint parser is very brittle: no ws! + SmallVector asmVals{v1,v2}; + auto asmStr = llvm::formatv(asmTp, llvm::format_hex(mask, /*width=*/2)).str(); + // clang-format on + + auto asmOp = b.create( + v1.getType(), asmVals, asmStr, asmCstr, false, false, asmDialectAttr); + return asmOp.getResult(0); +} + +Value mlir::x86vector::avx2::intrin::mm256UnpackLoPs(ImplicitLocOpBuilder &b, + Value v1, Value v2) { return b.create( v1, v2, ArrayRef{0, 8, 1, 9, 4, 12, 5, 13}); } -Value mlir::x86vector::avx2::mm256UnpackHiPs(ImplicitLocOpBuilder &b, Value v1, - Value v2) { +Value mlir::x86vector::avx2::intrin::mm256UnpackHiPs(ImplicitLocOpBuilder &b, + Value v1, Value v2) { return b.create( v1, v2, ArrayRef{2, 10, 3, 11, 6, 14, 7, 15}); } @@ -37,8 +60,9 @@ /// Takes an 8 bit mask, 2 bit for each position of a[0, 3) **and** b[0, 4): /// 0:127 | 128:255 /// b01 b23 C8 D8 | b01+4 b23+4 C8+4 D8+4 -Value mlir::x86vector::avx2::mm256ShufflePs(ImplicitLocOpBuilder &b, Value v1, - Value v2, int8_t mask) { +Value mlir::x86vector::avx2::intrin::mm256ShufflePs(ImplicitLocOpBuilder &b, + Value v1, Value v2, + int8_t mask) { int8_t b01, b23, b45, b67; MaskHelper::extractShuffle(mask, b01, b23, b45, b67); SmallVector shuffleMask{b01, b23, b45 + 8, b67 + 8, @@ -52,9 +76,8 @@ // a[0:127] or a[128:255] or b[0:127] or b[128:255] // 0 1 2 3 // imm[0:1] out of imm[4:7]. -Value mlir::x86vector::avx2::mm256Permute2f128Ps(ImplicitLocOpBuilder &b, - Value v1, Value v2, - int8_t mask) { +Value mlir::x86vector::avx2::intrin::mm256Permute2f128Ps( + ImplicitLocOpBuilder &b, Value v1, Value v2, int8_t mask) { SmallVector shuffleMask; auto appendToMask = [&](int8_t control) { if (control == 0) @@ -75,6 +98,18 @@ return b.create(v1, v2, shuffleMask); } +/// If bit i of `mask` is zero, take f32@i from v1 else take it from v2. +Value mlir::x86vector::avx2::intrin::mm256BlendPs(ImplicitLocOpBuilder &b, + Value v1, Value v2, + char mask) { + SmallVector shuffleMask; + for (int i = 0; i < 8; ++i) { + bool isSet = mask & (1 << i); + shuffleMask.push_back(!isSet ? i : i + 8); + } + return b.create(v1, v2, shuffleMask); +} + /// AVX2 4x8xf32-specific transpose lowering using a "C intrinsics" model. void mlir::x86vector::avx2::transpose4x8xf32(ImplicitLocOpBuilder &ib, MutableArrayRef vs) { @@ -118,14 +153,30 @@ Value T5 = mm256UnpackHiPs(ib, vs[4], vs[5]); Value T6 = mm256UnpackLoPs(ib, vs[6], vs[7]); Value T7 = mm256UnpackHiPs(ib, vs[6], vs[7]); - Value S0 = mm256ShufflePs(ib, T0, T2, MaskHelper::shuffle<1, 0, 1, 0>()); - Value S1 = mm256ShufflePs(ib, T0, T2, MaskHelper::shuffle<3, 2, 3, 2>()); - Value S2 = mm256ShufflePs(ib, T1, T3, MaskHelper::shuffle<1, 0, 1, 0>()); - Value S3 = mm256ShufflePs(ib, T1, T3, MaskHelper::shuffle<3, 2, 3, 2>()); - Value S4 = mm256ShufflePs(ib, T4, T6, MaskHelper::shuffle<1, 0, 1, 0>()); - Value S5 = mm256ShufflePs(ib, T4, T6, MaskHelper::shuffle<3, 2, 3, 2>()); - Value S6 = mm256ShufflePs(ib, T5, T7, MaskHelper::shuffle<1, 0, 1, 0>()); - Value S7 = mm256ShufflePs(ib, T5, T7, MaskHelper::shuffle<3, 2, 3, 2>()); + + using inline_asm::mm256BlendPs; + Value sh0 = mm256ShufflePs(ib, T0, T2, MaskHelper::shuffle<1, 0, 3, 2>()); + Value sh2 = mm256ShufflePs(ib, T1, T3, MaskHelper::shuffle<1, 0, 3, 2>()); + Value sh4 = mm256ShufflePs(ib, T4, T6, MaskHelper::shuffle<1, 0, 3, 2>()); + Value sh6 = mm256ShufflePs(ib, T5, T7, MaskHelper::shuffle<1, 0, 3, 2>()); + + Value S0 = + mm256BlendPs(ib, T0, sh0, MaskHelper::blend<0, 0, 1, 1, 0, 0, 1, 1>()); + Value S1 = + mm256BlendPs(ib, T2, sh0, MaskHelper::blend<1, 1, 0, 0, 1, 1, 0, 0>()); + Value S2 = + mm256BlendPs(ib, T1, sh2, MaskHelper::blend<0, 0, 1, 1, 0, 0, 1, 1>()); + Value S3 = + mm256BlendPs(ib, T3, sh2, MaskHelper::blend<1, 1, 0, 0, 1, 1, 0, 0>()); + Value S4 = + mm256BlendPs(ib, T4, sh4, MaskHelper::blend<0, 0, 1, 1, 0, 0, 1, 1>()); + Value S5 = + mm256BlendPs(ib, T6, sh4, MaskHelper::blend<1, 1, 0, 0, 1, 1, 0, 0>()); + Value S6 = + mm256BlendPs(ib, T5, sh6, MaskHelper::blend<0, 0, 1, 1, 0, 0, 1, 1>()); + Value S7 = + mm256BlendPs(ib, T7, sh6, MaskHelper::blend<1, 1, 0, 0, 1, 1, 0, 0>()); + vs[0] = mm256Permute2f128Ps(ib, S0, S4, MaskHelper::permute<2, 0>()); vs[1] = mm256Permute2f128Ps(ib, S1, S5, MaskHelper::permute<2, 0>()); vs[2] = mm256Permute2f128Ps(ib, S2, S6, MaskHelper::permute<2, 0>()); diff --git a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir --- a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir @@ -80,22 +80,17 @@ // AVX2-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32> // AVX2-NEXT: vector.shuffle {{.*}} [0, 8, 1, 9, 4, 12, 5, 13] : vector<8xf32>, vector<8xf32> // AVX2-NEXT: vector.shuffle {{.*}} [2, 10, 3, 11, 6, 14, 7, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 8, 9, 4, 5, 12, 13] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [2, 3, 10, 11, 6, 7, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32> - // AVX2-NEXT: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32> + // AVX2-COUNT-4: vector.shuffle {{.*}} [2, 3, 8, 9, 6, 7, 12, 13] : vector<8xf32>, vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0xcc", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-NEXT: llvm.inline_asm asm_dialect = intel "vblendps $0, $1, $2, 0x33", "=x,x,x" {{.*}} : (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + // AVX2-COUNT-4: vector.shuffle {{.*}} [0, 1, 2, 3, 8, 9, 10, 11] : vector<8xf32>, vector<8xf32> + // AVX2-COUNT-4: vector.shuffle {{.*}} [4, 5, 6, 7, 12, 13, 14, 15] : vector<8xf32>, vector<8xf32> %0 = vector.transpose %arg0, [1, 0] : vector<8x8xf32> to vector<8x8xf32> return %0 : vector<8x8xf32> } diff --git a/mlir/test/Integration/Dialect/LLVMIR/CPU/X86/test-inline-asm-vector.mlir b/mlir/test/Integration/Dialect/LLVMIR/CPU/X86/test-inline-asm-vector.mlir new file mode 100644 --- /dev/null +++ b/mlir/test/Integration/Dialect/LLVMIR/CPU/X86/test-inline-asm-vector.mlir @@ -0,0 +1,56 @@ +// RUN: mlir-opt %s -convert-vector-to-llvm | \ +// RUN: mlir-cpu-runner -e entry_point_with_all_constants -entry-point-result=void \ +// RUN: -shared-libs=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext + +module { + llvm.func @function_to_run(%a: vector<8xf32>, %b: vector<8xf32>) { + // CHECK: ( 8, 10, 12, 14, 16, 18, 20, 22 ) + %r0 = llvm.inline_asm asm_dialect = intel + "vaddps $0, $1, $2", "=x,x,x" %a, %b: + (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + vector.print %r0: vector<8xf32> + + // vblendps implemented with inline_asm. + // CHECK: ( 0, 1, 10, 11, 4, 5, 14, 15 ) + %r1 = llvm.inline_asm asm_dialect = intel + "vblendps $0, $1, $2, 0xCC", "=x,x,x" %a, %b: + (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + vector.print %r1: vector<8xf32> + + // vblendps 0xCC via vector.shuffle (emulates clang intrinsics impl) + // CHECK: ( 0, 1, 10, 11, 4, 5, 14, 15 ) + %r2 = vector.shuffle %a, %b[0, 1, 10, 11, 4, 5, 14, 15] + : vector<8xf32>, vector<8xf32> + vector.print %r2: vector<8xf32> + + // vblendps 0x33 implemented with inline_asm. + // CHECK: ( 8, 9, 2, 3, 12, 13, 6, 7 ) + %r3 = llvm.inline_asm asm_dialect = intel + "vblendps $0, $1, $2, 0x33", "=x,x,x" %a, %b: + (vector<8xf32>, vector<8xf32>) -> vector<8xf32> + vector.print %r3: vector<8xf32> + + // vblendps 0x33 via vector.shuffle (emulates clang intrinsics impl) + // CHECK: ( 8, 9, 2, 3, 12, 13, 6, 7 ) + %r4 = vector.shuffle %a, %b[8, 9, 2, 3, 12, 13, 6, 7] + : vector<8xf32>, vector<8xf32> + vector.print %r4: vector<8xf32> + + llvm.return + } + + // Solely exists to prevent inlining and get the expected assembly. + llvm.func @entry_point(%a: vector<8xf32>, %b: vector<8xf32>) { + llvm.call @function_to_run(%a, %b) : (vector<8xf32>, vector<8xf32>) -> () + llvm.return + } + + llvm.func @entry_point_with_all_constants() { + %a = llvm.mlir.constant(dense<[0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]> + : vector<8xf32>) : vector<8xf32> + %b = llvm.mlir.constant(dense<[8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0]> + : vector<8xf32>) : vector<8xf32> + llvm.call @function_to_run(%a, %b) : (vector<8xf32>, vector<8xf32>) -> () + llvm.return + } +} diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp --- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp +++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp @@ -13,6 +13,8 @@ #include "mlir/Dialect/Linalg/IR/LinalgOps.h" #include "mlir/Dialect/Linalg/Passes.h" #include "mlir/Dialect/Linalg/Transforms/Transforms.h" +// TODO: Can we make this optional depending on whether we use the InlineAsmOp? +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/SCF.h" #include "mlir/Dialect/StandardOps/IR/Ops.h" @@ -200,6 +202,11 @@ llvm::cl::desc("Lower vector.transpose to avx2-specific patterns"), llvm::cl::init(false)}; + void getDependentDialects(DialectRegistry ®istry) const { + // TODO: make this conditional on whether we use the InlineAsmOp? + registry.insert(); + } + void runOnFunction() override { RewritePatternSet patterns(&getContext()); diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -483,6 +483,7 @@ deps = [ "//mlir:Affine", "//mlir:Analysis", + "//mlir:LLVMDialect", "//mlir:LinalgOps", "//mlir:LinalgTransforms", "//mlir:MemRefDialect", @@ -492,6 +493,7 @@ "//mlir:TransformUtils", "//mlir:VectorOps", "//mlir:VectorToSCF", + "//mlir:X86Vector", ], )