[mlir] squash LLVM_AVX512 dialect into AVX512

[ftynse]

Summary

The dialect separation was introduced to demarkate ops operating in different
type systems. This is no longer the case after the LLVM dialect has migrated to
using built-in vector types, so the original reason for separation is no longer
valid. Squash the two dialects into one.

The code size decrease isn't quite large: the ops originally in LLVM_AVX512 are
preserved because they match LLVM IR intrinsics specialized for vector element
bitwidth. However, it is still conceptually beneficial to have only one
dialect. I originally considered to use Tablegen multiclasses to define both
the type-polymorphic op and its two intrinsic-related instantiations, but
decided against it given both the complexity of the required Tablegen input and
its dissimilarity with the rest of ODS-defined ops, both potentially resulting
in very poor maintainability.

Depends On D98327

[ftynse]

Slightly more cmake

[nicolasvasilache]

Nice!

mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h

-//===- ConvertAVX512ToLLVM.h - Conversion Patterns from AVX512 to LLVM ----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
-#define MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
-
-namespace mlir {
-
-class LLVMTypeConverter;
-class OwningRewritePatternList;
-
-/// Collect a set of patterns to convert from the AVX512 dialect to LLVM.
-void populateAVX512ToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                            OwningRewritePatternList &patterns);
-
-} // namespace mlir
-
-#endif // MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_

mlir/include/mlir/Dialect/AVX512/AVX512.td

 //===-- AVX512Ops.td - AVX512 dialect operation definitions *- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the basic operations for the AVX512 dialect.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef AVX512_OPS
 #define AVX512_OPS
 
 include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
 
 //===----------------------------------------------------------------------===//
 // AVX512 dialect definition
 //===----------------------------------------------------------------------===//
 
 def AVX512_Dialect : Dialect {
   let name = "avx512";
   let cppNamespace = "::mlir::avx512";
 }
 
 //===----------------------------------------------------------------------===//
 // AVX512 op definitions
 //===----------------------------------------------------------------------===//
 
 class AVX512_Op<string mnemonic, list<OpTrait> traits = []> :
   Op<AVX512_Dialect, mnemonic, traits> {}
 
+class AVX512_IntrOp<string mnemonic, int numResults, list<OpTrait> traits = []> :
+  LLVM_IntrOpBase<AVX512_Dialect, "intr." # mnemonic,
+                  "x86_avx512_" # !subst(".", "_", mnemonic),
+                  [], [], traits, numResults>;
+
+// Defined by first result overload. May have to be extended for other
+// instructions in the future.
+class AVX512_IntrOverloadedOp<string mnemonic,
+                              list<OpTrait> traits = []> :
+  LLVM_IntrOpBase<AVX512_Dialect, "intr." # mnemonic,
+                  "x86_avx512_" # !subst(".", "_", mnemonic),
+                  /*list<int> overloadedResults=*/[0],
+                  /*list<int> overloadedOperands=*/[],
+                  traits, /*numResults=*/1>;
+//----------------------------------------------------------------------------//
+// MaskCompressOp
+//----------------------------------------------------------------------------//
+
 def MaskCompressOp : AVX512_Op<"mask.compress", [NoSideEffect,
   // TODO: Support optional arguments in `AllTypesMatch`. "type($src)" could
   // then be removed from assemblyFormat.
   AllTypesMatch<["a", "dst"]>,
   TypesMatchWith<"`k` has the same number of bits as elements in `dst`",
                  "dst", "k",
                  "VectorType::get({$_self.cast<VectorType>().getShape()[0]}, "
                  "IntegerType::get($_self.getContext(), 1))">]> {
let arguments = (ins VectorOfLengthAndType<[16, 8],
                                                   [F32, I32, F64, I64]>>:$src,
                    OptionalAttr<ElementsAttr>:$constant_src);
   let results = (outs VectorOfLengthAndType<[16, 8],
                                             [F32, I32, F64, I64]>:$dst);
   let assemblyFormat = "$k `,` $a (`,` $src^)? attr-dict"
                        " `:` type($dst) (`,` type($src)^)?";
 }
 
+def MaskCompressIntrOp : AVX512_IntrOverloadedOp<"mask.compress", [
+  NoSideEffect,
+  AllTypesMatch<["a", "src", "res"]>,
+  TypesMatchWith<"`k` has the same number of bits as elements in `res`",
+                 "res", "k",
+                 "VectorType::get({$_self.cast<VectorType>().getShape()[0]}, "
+                 "IntegerType::get($_self.getContext(), 1))">]> {
+  let arguments = (ins VectorOfLengthAndType<[16, 8],
+                                             [F32, I32, F64, I64]>:$a,
+                   VectorOfLengthAndType<[16, 8],
+                                         [F32, I32, F64, I64]>:$src,
+                   VectorOfLengthAndType<[16, 8],
+                                         [I1]>:$k);
+}
+
+//----------------------------------------------------------------------------//
+// MaskRndScaleOp
+//----------------------------------------------------------------------------//
+
 def MaskRndScaleOp : AVX512_Op<"mask.rndscale", [NoSideEffect,
   AllTypesMatch<["src", "a", "dst"]>,
   TypesMatchWith<"imm has the same number of bits as elements in dst",
                  "dst", "imm",
                  "IntegerType::get($_self.getContext(), "
                  "($_self.cast<VectorType>().getShape()[0]))">]> {
   let summary = "Masked roundscale op";
   let description = [{
let arguments = (ins VectorOfLengthAndType<[16, 8], [F32, F64]>:$src,
                    // TODO: figure rounding out (optional operand?).
                    I32:$rounding
             );
   let results = (outs VectorOfLengthAndType<[16, 8], [F32, F64]>:$dst);
   let assemblyFormat =
     "$src `,` $k `,` $a `,` $imm `,` $rounding attr-dict `:` type($dst)";
 }
 
+def MaskRndScalePSIntrOp : AVX512_IntrOp<"mask.rndscale.ps.512", 1, [
+  NoSideEffect,
+  AllTypesMatch<["src", "a", "res"]>]> {
+  let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$src,
+                   I32:$k,
+                   VectorOfLengthAndType<[16], [F32]>:$a,
+                   I16:$imm,
+                   LLVM_Type:$rounding);
+}
+
+def MaskRndScalePDIntrOp : AVX512_IntrOp<"mask.rndscale.pd.512", 1, [
+  NoSideEffect,
+  AllTypesMatch<["src", "a", "res"]>]> {
+  let arguments = (ins VectorOfLengthAndType<[8], [F64]>:$src,
+                   I32:$k,
+                   VectorOfLengthAndType<[8], [F64]>:$a,
+                   I8:$imm,
+                   LLVM_Type:$rounding);
+}
+
+//----------------------------------------------------------------------------//
+// MaskScaleFOp
+//----------------------------------------------------------------------------//
+
 def MaskScaleFOp : AVX512_Op<"mask.scalef", [NoSideEffect,
   AllTypesMatch<["src", "a", "b", "dst"]>,
   TypesMatchWith<"k has the same number of bits as elements in dst",
                  "dst", "k",
                  "IntegerType::get($_self.getContext(), "
                  "($_self.cast<VectorType>().getShape()[0]))">]> {
   let summary = "ScaleF op";
   let description = [{
let arguments = (ins VectorOfLengthAndType<[16, 8], [F32, F64]>:$src,
                    I32:$rounding
             );
   let results = (outs VectorOfLengthAndType<[16, 8], [F32, F64]>:$dst);
   // Fully specified by traits.
   let assemblyFormat =
     "$src `,` $a `,` $b `,` $k `,` $rounding attr-dict `:` type($dst)";
 }
 
+def MaskScaleFPSIntrOp : AVX512_IntrOp<"mask.scalef.ps.512", 1, [
+  NoSideEffect,
+  AllTypesMatch<["src", "a", "b", "res"]>]> {
+  let arguments = (ins VectorOfLengthAndType<[16], [F32]>:$src,
+                   VectorOfLengthAndType<[16], [F32]>:$a,
+                   VectorOfLengthAndType<[16], [F32]>:$b,
+                   I16:$k,
+                   LLVM_Type:$rounding);
+}
+
+def MaskScaleFPDIntrOp : AVX512_IntrOp<"mask.scalef.pd.512", 1, [
+  NoSideEffect,
+  AllTypesMatch<["src", "a", "b", "res"]>]> {
+  let arguments = (ins VectorOfLengthAndType<[8], [F64]>:$src,
+                   VectorOfLengthAndType<[8], [F64]>:$a,
+                   VectorOfLengthAndType<[8], [F64]>:$b,
+                   I8:$k,
+                   LLVM_Type:$rounding);
+}
+
+//----------------------------------------------------------------------------//
+// Vp2IntersectOp
+//----------------------------------------------------------------------------//
+
 def Vp2IntersectOp : AVX512_Op<"vp2intersect", [NoSideEffect,
   AllTypesMatch<["a", "b"]>,
   TypesMatchWith<"k1 has the same number of bits as elements in a",
                  "a", "k1",
                  "VectorType::get({$_self.cast<VectorType>().getShape()[0]}, "
                  "IntegerType::get($_self.getContext(), 1))">,
   TypesMatchWith<"k2 has the same number of bits as elements in b",
                  // Should use `b` instead of `a`, but that would require
let arguments = (ins VectorOfLengthAndType<[16, 8], [I32, I64]>:$a,
                    );
   let results = (outs VectorOfLengthAndType<[16, 8], [I1]>:$k1,
                  VectorOfLengthAndType<[16, 8], [I1]>:$k2
                  );
   let assemblyFormat =
     "$a `,` $b attr-dict `:` type($a)";
 }
 
+def Vp2IntersectDIntrOp : AVX512_IntrOp<"vp2intersect.d.512", 2, [
+  NoSideEffect]> {
+  let arguments = (ins VectorOfLengthAndType<[16], [I32]>:$a,
+                   VectorOfLengthAndType<[16], [I32]>:$b);
+}
+
+def Vp2IntersectQIntrOp : AVX512_IntrOp<"vp2intersect.q.512", 2, [
+  NoSideEffect]> {
+  let arguments = (ins VectorOfLengthAndType<[8], [I64]>:$a,
+                   VectorOfLengthAndType<[8], [I64]>:$b);
+}
+
 #endif // AVX512_OPS

mlir/include/mlir/Dialect/AVX512/CMakeLists.txt

 add_mlir_dialect(AVX512 avx512)
 add_mlir_doc(AVX512 -gen-dialect-doc AVX512 Dialects/)
+
+set(LLVM_TARGET_DEFINITIONS AVX512.td)
+mlir_tablegen(AVX512Conversions.inc -gen-llvmir-conversions)
+add_public_tablegen_target(MLIRAVX512ConversionsIncGen)

mlir/include/mlir/Dialect/AVX512/Transforms.h

+//===- Transforms.h - AVX512 Dialect Transformation Entrypoints -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_AVX512_TRANSFORMS_H
+#define MLIR_DIALECT_AVX512_TRANSFORMS_H
+
+namespace mlir {
+
+class LLVMConversionTarget;
+class LLVMTypeConverter;
+class OwningRewritePatternList;
+
+/// Collect a set of patterns to lower AVX512 ops to ops that map to LLVM
+/// intrinsics.
+void populateAVX512LegalizeForLLVMExportPatterns(
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+
+/// Configure the target to support lowering AVX512 ops to ops that map to LLVM
+/// intrinsics.
+void configureAVX512LegalizeForExportTarget(LLVMConversionTarget &target);
+
+} // namespace mlir
+
+#endif // MLIR_DIALECT_AVX512_TRANSFORMS_H

mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt

 add_public_tablegen_target(MLIRNVVMConversionsIncGen)
 
 add_mlir_dialect(ROCDLOps rocdl)
 add_mlir_doc(ROCDLOps -gen-dialect-doc ROCDLDialect Dialects/)
 set(LLVM_TARGET_DEFINITIONS ROCDLOps.td)
 mlir_tablegen(ROCDLConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRROCDLConversionsIncGen)
 
-add_mlir_dialect(LLVMAVX512 llvm_avx512 LLVMAVX512)
-add_mlir_doc(LLVMAVX512 -gen-dialect-doc LLVMAVX512 Dialects/)
-set(LLVM_TARGET_DEFINITIONS LLVMAVX512.td)
-mlir_tablegen(LLVMAVX512Conversions.inc -gen-llvmir-conversions)
-add_public_tablegen_target(MLIRLLVMAVX512ConversionsIncGen)
-
 add_mlir_dialect(LLVMArmSVE llvm_arm_sve LLVMArmSVE)
 add_mlir_doc(LLVMArmSVE -gen-dialect-doc LLVMArmSve Dialects/)
 set(LLVM_TARGET_DEFINITIONS LLVMArmSVE.td)
 mlir_tablegen(LLVMArmSVEConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRLLVMArmSVEConversionsIncGen)

mlir/include/mlir/Dialect/LLVMIR/LLVMAVX512.td

-//===-- LLVMAVX512.td - LLVMAVX512 dialect op definitions --*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the basic operations for the LLVMAVX512 dialect.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVMIR_AVX512_OPS
-#define LLVMIR_AVX512_OPS
-
-include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
-
-//===----------------------------------------------------------------------===//
-// LLVMAVX512 dialect definition
-//===----------------------------------------------------------------------===//
-
-def LLVMAVX512_Dialect : Dialect {
-  let name = "llvm_avx512";
-  let cppNamespace = "::mlir::LLVM";
-}
-
-//----------------------------------------------------------------------------//
-// MLIR LLVM AVX512 intrinsics using the MLIR LLVM Dialect type system
-//----------------------------------------------------------------------------//
-
-class LLVMAVX512_IntrOp<string mnemonic, int numResults, list<OpTrait> traits = []> :
-  LLVM_IntrOpBase<LLVMAVX512_Dialect, mnemonic,
-                  "x86_avx512_" # !subst(".", "_", mnemonic),
-                  [], [], traits, numResults>;
-
-// Defined by first result overload. May have to be extended for other
-// instructions in the future.
-class LLVMAVX512_IntrOverloadedOp<string mnemonic,
-                                  list<OpTrait> traits = []> :
-  LLVM_IntrOpBase<LLVMAVX512_Dialect, mnemonic,
-                  "x86_avx512_" # !subst(".", "_", mnemonic),
-                  /*list<int> overloadedResults=*/[0],
-                  /*list<int> overloadedOperands=*/[],
-                  traits, /*numResults=*/1>;
-
-def LLVM_x86_avx512_mask_rndscale_ps_512 :
-  LLVMAVX512_IntrOp<"mask.rndscale.ps.512", 1>,
-  Arguments<(ins LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type)>;
-
-def LLVM_x86_avx512_mask_rndscale_pd_512 :
-  LLVMAVX512_IntrOp<"mask.rndscale.pd.512", 1>,
-  Arguments<(ins LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type)>;
-
-def LLVM_x86_avx512_mask_scalef_ps_512 :
-  LLVMAVX512_IntrOp<"mask.scalef.ps.512", 1>,
-  Arguments<(ins LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type)>;
-
-def LLVM_x86_avx512_mask_scalef_pd_512 :
-  LLVMAVX512_IntrOp<"mask.scalef.pd.512", 1>,
-  Arguments<(ins LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type, LLVM_Type)>;
-
-def LLVM_x86_avx512_mask_compress :
-  LLVMAVX512_IntrOverloadedOp<"mask.compress">,
-  Arguments<(ins LLVM_Type, LLVM_Type, LLVM_Type)>;
-
-def LLVM_x86_avx512_vp2intersect_d_512 :
-  LLVMAVX512_IntrOp<"vp2intersect.d.512", 2>,
-  Arguments<(ins LLVM_Type, LLVM_Type)>;
-
-def LLVM_x86_avx512_vp2intersect_q_512 :
-  LLVMAVX512_IntrOp<"vp2intersect.q.512", 2>,
-  Arguments<(ins LLVM_Type, LLVM_Type)>;
-
-#endif // AVX512_OPS

mlir/include/mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h

-//===- LLVMAVX512Dialect.h - MLIR Dialect for LLVMAVX512 --------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the Target dialect for LLVMAVX512 in MLIR.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_DIALECT_LLVMIR_LLVMAVX512DIALECT_H_
-#define MLIR_DIALECT_LLVMIR_LLVMAVX512DIALECT_H_
-
-#include "mlir/IR/Dialect.h"
-#include "mlir/IR/OpDefinition.h"
-
-#define GET_OP_CLASSES
-#include "mlir/Dialect/LLVMIR/LLVMAVX512.h.inc"
-
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h.inc"
-
-#endif // MLIR_DIALECT_LLVMIR_LLVMAVX512DIALECT_H_

mlir/include/mlir/InitAllDialects.h

 
 #include "mlir/Dialect/AVX512/AVX512Dialect.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
 #include "mlir/Dialect/Async/IR/Async.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMArmSVEDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
inline void registerAllDialects(DialectRegistry &registry) {
   // clang-format off
   registry.insert<acc::OpenACCDialect,
                   AffineDialect,
                   arm_neon::ArmNeonDialect,
                   async::AsyncDialect,
                   avx512::AVX512Dialect,
                   complex::ComplexDialect,
                   gpu::GPUDialect,
-                  LLVM::LLVMAVX512Dialect,
                   LLVM::LLVMDialect,
                   LLVM::LLVMArmSVEDialect,
                   linalg::LinalgDialect,
                   math::MathDialect,
                   scf::SCFDialect,
                   omp::OpenMPDialect,
                   pdl::PDLDialect,
                   pdl_interp::PDLInterpDialect,

mlir/include/mlir/Target/LLVMIR/Dialect/AVX512/AVX512ToLLVMIRTranslation.h

+//===- AVX512ToLLVMIRTranslation.h - AVX512 to LLVM IR ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides registration calls for AVX512 dialect to LLVM IR
+// translation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVMIR_DIALECT_AVX512_AVX512TOLLVMIRTRANSLATION_H
+#define MLIR_TARGET_LLVMIR_DIALECT_AVX512_AVX512TOLLVMIRTRANSLATION_H
+
+namespace mlir {
+
+class DialectRegistry;
+class MLIRContext;
+
+/// Register the AVX512 dialect and the translation from it to the LLVM IR
+/// in the given registry;
+void registerAVX512DialectTranslation(DialectRegistry &registry);
+
+/// Register the AVX512 dialect and the translation from it in the registry
+/// associated with the given context.
+void registerAVX512DialectTranslation(MLIRContext &context);
+
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVMIR_DIALECT_AVX512_AVX512TOLLVMIRTRANSLATION_H

mlir/include/mlir/Target/LLVMIR/Dialect/All.h

 //===- All.h - MLIR To LLVM IR Translation Registration ---------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines a helper to register the translations of all suitable
 // dialects to LLVM IR.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef MLIR_TARGET_LLVMIR_DIALECT_ALL_H
 #define MLIR_TARGET_LLVMIR_DIALECT_ALL_H
 
+#include "mlir/Target/LLVMIR/Dialect/AVX512/AVX512ToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/ArmNeon/ArmNeonToLLVMIRTranslation.h"
-#include "mlir/Target/LLVMIR/Dialect/LLVMAVX512/LLVMAVX512ToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/LLVMArmSVE/LLVMArmSVEToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
 
 namespace mlir {
 class DialectRegistry;
 
 /// Registers all dialects that can be translated to LLVM IR and the
 /// corresponding translation interfaces.
 static inline void registerAllToLLVMIRTranslations(DialectRegistry &registry) {
   registerArmNeonDialectTranslation(registry);
-  registerLLVMAVX512DialectTranslation(registry);
+  registerAVX512DialectTranslation(registry);
   registerLLVMArmSVEDialectTranslation(registry);
   registerLLVMDialectTranslation(registry);
   registerNVVMDialectTranslation(registry);
   registerOpenMPDialectTranslation(registry);
   registerROCDLDialectTranslation(registry);
 }
 } // namespace mlir
 
 #endif // MLIR_TARGET_LLVMIR_DIALECT_ALL_H

mlir/include/mlir/Target/LLVMIR/Dialect/LLVMAVX512/LLVMAVX512ToLLVMIRTranslation.h

-//===- LLVMAVX512ToLLVMIRTranslation.h - LLVMAVX512 to LLVM IR --*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This provides registration calls for LLVMAVX512 dialect to LLVM IR
-// translation.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_TARGET_LLVMIR_DIALECT_LLVMAVX512_LLVMAVX512TOLLVMIRTRANSLATION_H
-#define MLIR_TARGET_LLVMIR_DIALECT_LLVMAVX512_LLVMAVX512TOLLVMIRTRANSLATION_H
-
-namespace mlir {
-
-class DialectRegistry;
-class MLIRContext;
-
-/// Register the LLVMAVX512 dialect and the translation from it to the LLVM IR
-/// in the given registry;
-void registerLLVMAVX512DialectTranslation(DialectRegistry &registry);
-
-/// Register the LLVMAVX512 dialect and the translation from it in the registry
-/// associated with the given context.
-void registerLLVMAVX512DialectTranslation(MLIRContext &context);
-
-} // namespace mlir
-
-#endif // MLIR_TARGET_LLVMIR_DIALECT_LLVMAVX512_LLVMAVX512TOLLVMIRTRANSLATION_H

mlir/lib/Conversion/AVX512ToLLVM/CMakeLists.txt

-add_mlir_conversion_library(MLIRAVX512ToLLVM
-  ConvertAVX512ToLLVM.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/AVX512ToLLVM
-
-  DEPENDS
-  MLIRConversionPassIncGen
-
-  LINK_COMPONENTS
-  Core
-
-  LINK_LIBS PUBLIC
-  MLIRAVX512
-  MLIRLLVMAVX512
-  MLIRLLVMIR
-  MLIRStandardToLLVM
-  MLIRTransforms
-  )

mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp

-//===- ConvertAVX512ToLLVM.cpp - Convert AVX512 to the LLVM dialect -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
-
-#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
-#include "mlir/Dialect/AVX512/AVX512Dialect.h"
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/Dialect/Vector/VectorOps.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/PatternMatch.h"
-
-using namespace mlir;
-using namespace mlir::vector;
-using namespace mlir::avx512;
-
-template <typename OpTy>
-static Type getSrcVectorElementType(Operation *op) {
-  return cast<OpTy>(op)
-      .src()
-      .getType()
-      .template cast<VectorType>()
-      .getElementType();
-}
-
-namespace {
-
-// TODO: turn these into simpler declarative templated patterns when we've had
-// enough.
-struct MaskRndScaleOp512Conversion : public ConvertToLLVMPattern {
-  explicit MaskRndScaleOp512Conversion(MLIRContext *context,
-                                       LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(MaskRndScaleOp::getOperationName(), context,
-                             typeConverter) {}
-
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    Type elementType = getSrcVectorElementType<MaskRndScaleOp>(op);
-    if (elementType.isF32())
-      return LLVM::detail::oneToOneRewrite(
-          op, LLVM::x86_avx512_mask_rndscale_ps_512::getOperationName(),
-          operands, *getTypeConverter(), rewriter);
-    if (elementType.isF64())
-      return LLVM::detail::oneToOneRewrite(
-          op, LLVM::x86_avx512_mask_rndscale_pd_512::getOperationName(),
-          operands, *getTypeConverter(), rewriter);
-    return failure();
-  }
-};
-
-struct MaskCompressOpConversion
-    : public ConvertOpToLLVMPattern<MaskCompressOp> {
-  using ConvertOpToLLVMPattern<MaskCompressOp>::ConvertOpToLLVMPattern;
-
-  LogicalResult
-  matchAndRewrite(MaskCompressOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    MaskCompressOp::Adaptor adaptor(operands);
-    auto opType = adaptor.a().getType();
-
-    Value src;
-    if (op.src()) {
-      src = adaptor.src();
-    } else if (op.constant_src()) {
-      src = rewriter.create<ConstantOp>(op.getLoc(), opType,
-                                        op.constant_srcAttr());
-    } else {
-      Attribute zeroAttr = rewriter.getZeroAttr(opType);
-      src = rewriter.create<ConstantOp>(op->getLoc(), opType, zeroAttr);
-    }
-
-    rewriter.replaceOpWithNewOp<LLVM::x86_avx512_mask_compress>(
-        op, opType, adaptor.a(), src, adaptor.k());
-
-    return success();
-  }
-};
-
-struct ScaleFOp512Conversion : public ConvertToLLVMPattern {
-  explicit ScaleFOp512Conversion(MLIRContext *context,
-                                 LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(MaskScaleFOp::getOperationName(), context,
-                             typeConverter) {}
-
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    Type elementType = getSrcVectorElementType<MaskScaleFOp>(op);
-    if (elementType.isF32())
-      return LLVM::detail::oneToOneRewrite(
-          op, LLVM::x86_avx512_mask_scalef_ps_512::getOperationName(), operands,
-          *getTypeConverter(), rewriter);
-    if (elementType.isF64())
-      return LLVM::detail::oneToOneRewrite(
-          op, LLVM::x86_avx512_mask_scalef_pd_512::getOperationName(), operands,
-          *getTypeConverter(), rewriter);
-    return failure();
-  }
-};
-
-struct Vp2IntersectOp512Conversion
-    : public ConvertOpToLLVMPattern<Vp2IntersectOp> {
-  explicit Vp2IntersectOp512Conversion(MLIRContext *context,
-                                       LLVMTypeConverter &typeConverter)
-      : ConvertOpToLLVMPattern<Vp2IntersectOp>(typeConverter) {}
-
-  LogicalResult
-  matchAndRewrite(Vp2IntersectOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    Type elementType =
-        op.a().getType().template cast<VectorType>().getElementType();
-    if (elementType.isInteger(32))
-      return LLVM::detail::oneToOneRewrite(
-          op, LLVM::x86_avx512_vp2intersect_d_512::getOperationName(), operands,
-          *getTypeConverter(), rewriter);
-    if (elementType.isInteger(64))
-      return LLVM::detail::oneToOneRewrite(
-          op, LLVM::x86_avx512_vp2intersect_q_512::getOperationName(), operands,
-          *getTypeConverter(), rewriter);
-    return failure();
-  }
-};
-} // namespace
-
-/// Populate the given list with patterns that convert from AVX512 to LLVM.
-void mlir::populateAVX512ToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  // clang-format off
-  patterns.insert<MaskRndScaleOp512Conversion,
-                  ScaleFOp512Conversion,
-                  Vp2IntersectOp512Conversion>(&converter.getContext(),
-                                               converter);
-  patterns.insert<MaskCompressOpConversion>(converter);
-  // clang-format on
-}

mlir/lib/Conversion/CMakeLists.txt

 add_subdirectory(AffineToStandard)
 add_subdirectory(AsyncToLLVM)
-add_subdirectory(AVX512ToLLVM)
 add_subdirectory(ComplexToLLVM)
 add_subdirectory(GPUCommon)
 add_subdirectory(GPUToNVVM)
 add_subdirectory(GPUToROCDL)
 add_subdirectory(GPUToSPIRV)
 add_subdirectory(GPUToVulkan)
 add_subdirectory(LinalgToLLVM)
 add_subdirectory(LinalgToSPIRV)

mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt

add_mlir_conversion_library(MLIRVectorToLLVM
   intrinsics_gen
 
   LINK_COMPONENTS
   Core
 
   LINK_LIBS PUBLIC
   MLIRArmNeon
   MLIRAVX512
-  MLIRAVX512ToLLVM
-  MLIRLLVMAVX512
+  MLIRAVX512Transforms
   MLIRArmSVE
   MLIRArmSVEToLLVM
   MLIRLLVMArmSVE
   MLIRLLVMIR
   MLIRStandardToLLVM
   MLIRTargetLLVMIRExport
   MLIRTransforms
   MLIRVector
   )

mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp

 //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 
 #include "../PassDetail.h"
 
-#include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
 #include "mlir/Conversion/ArmSVEToLLVM/ArmSVEToLLVM.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include "mlir/Dialect/AVX512/AVX512Dialect.h"
+#include "mlir/Dialect/AVX512/Transforms.h"
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/ArmSVE/ArmSVEDialect.h"
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMArmSVEDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace mlir::vector;
struct LowerVectorToLLVMPass
   // Override explicitly to allow conditional dialect dependence.
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<LLVM::LLVMDialect>();
     if (enableArmNeon)
       registry.insert<arm_neon::ArmNeonDialect>();
     if (enableArmSVE)
       registry.insert<LLVM::LLVMArmSVEDialect>();
     if (enableAVX512)
-      registry.insert<LLVM::LLVMAVX512Dialect>();
+      registry.insert<avx512::AVX512Dialect>();
   }
   void runOnOperation() override;
 };
 } // namespace
 
 void LowerVectorToLLVMPass::runOnOperation() {
   // Perform progressive lowering of operations on slices and
   // all contraction operations. Also applies folding and DCE.
if (enableArmSVE) {
     target.addDynamicallyLegalOp<CallOp, CallIndirectOp, ReturnOp>(
         [hasScalableVectorType](Operation *op) {
           return !hasScalableVectorType(op->getOperandTypes()) &&
                  !hasScalableVectorType(op->getResultTypes());
         });
     populateArmSVEToLLVMConversionPatterns(converter, patterns);
   }
   if (enableAVX512) {
-    target.addLegalDialect<LLVM::LLVMAVX512Dialect>();
-    target.addIllegalDialect<avx512::AVX512Dialect>();
-    populateAVX512ToLLVMConversionPatterns(converter, patterns);
+    configureAVX512LegalizeForExportTarget(target);
+    populateAVX512LegalizeForLLVMExportPatterns(converter, patterns);
   }
 
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
     signalPassFailure();
 }
 
 std::unique_ptr<OperationPass<ModuleOp>>
 mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
   return std::make_unique<LowerVectorToLLVMPass>(options);
 }

mlir/lib/Dialect/AVX512/CMakeLists.txt

-add_mlir_dialect_library(MLIRAVX512
-  IR/AVX512Dialect.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AVX512
-
-  DEPENDS
-  MLIRAVX512IncGen
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRSideEffectInterfaces
-  )
+add_subdirectory(IR)
+add_subdirectory(Transforms)

mlir/lib/Dialect/AVX512/IR/AVX512Dialect.cpp

 //===- AVX512Ops.cpp - MLIR AVX512 ops implementation ---------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the AVX512 dialect and its operations.
 //
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/AVX512/AVX512Dialect.h"
-#include "mlir/Dialect/Vector/VectorOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TypeUtilities.h"
 
 using namespace mlir;
 
 void avx512::AVX512Dialect::initialize() {
   addOperations<

mlir/lib/Dialect/AVX512/IR/CMakeLists.txt

 add_mlir_dialect_library(MLIRAVX512
-  IR/AVX512Dialect.cpp
+  AVX512Dialect.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AVX512
 
   DEPENDS
   MLIRAVX512IncGen
 
   LINK_LIBS PUBLIC
   MLIRIR
+  MLIRLLVMIR
   MLIRSideEffectInterfaces
   )

mlir/lib/Dialect/AVX512/Transforms/CMakeLists.txt

+add_mlir_dialect_library(MLIRAVX512Transforms
+  LegalizeForLLVMExport.cpp
+
+  DEPENDS
+  MLIRAVX512ConversionsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRAVX512
+  MLIRIR
+  MLIRLLVMIR
+  )

mlir/lib/Dialect/AVX512/Transforms/LegalizeForLLVMExport.cpp

+//===- LegalizeForLLVMExport.cpp - Prepare AVX512 for LLVM translation ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/AVX512/Transforms.h"
+
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
+#include "mlir/Dialect/AVX512/AVX512Dialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/PatternMatch.h"
+
+using namespace mlir;
+using namespace mlir::avx512;
+
+/// Extracts the "main" vector element type from the given AVX512 operation.
+template <typename OpTy>
+static Type getSrcVectorElementType(OpTy op) {
+  return op.src().getType().template cast<VectorType>().getElementType();
+}
+template <>
+Type getSrcVectorElementType(Vp2IntersectOp op) {
+  return op.a().getType().template cast<VectorType>().getElementType();
+}
+
+namespace {
+/// Base conversion for AVX512 ops that can be lowered to one of the two
+/// intrinsics based on the bitwidth of their "main" vector element type. This
+/// relies on the to-LLVM-dialect conversion helpers to correctly pack the
+/// results of multi-result intrinsic ops.
+template <typename OpTy, typename Intr32OpTy, typename Intr64OpTy>
+struct LowerToIntrinsic : public OpConversionPattern<OpTy> {
+  explicit LowerToIntrinsic(LLVMTypeConverter &converter)
+      : OpConversionPattern<OpTy>(converter, &converter.getContext()) {}
+
+  LLVMTypeConverter &getTypeConverter() const {
+    return *static_cast<LLVMTypeConverter *>(
+        OpConversionPattern<OpTy>::getTypeConverter());
+  }
+
+  LogicalResult
+  matchAndRewrite(OpTy op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    Type elementType = getSrcVectorElementType<OpTy>(op);
+    unsigned bitwidth = elementType.getIntOrFloatBitWidth();
+    if (bitwidth == 32)
+      return LLVM::detail::oneToOneRewrite(op, Intr32OpTy::getOperationName(),
+                                           operands, getTypeConverter(),
+                                           rewriter);
+    if (bitwidth == 64)
+      return LLVM::detail::oneToOneRewrite(op, Intr64OpTy::getOperationName(),
+                                           operands, getTypeConverter(),
+                                           rewriter);
+    return rewriter.notifyMatchFailure(
+        op, "expected 'src' to be either f32 or f64");
+  }
+};
+
+struct MaskCompressOpConversion
+    : public ConvertOpToLLVMPattern<MaskCompressOp> {
+  using ConvertOpToLLVMPattern<MaskCompressOp>::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(MaskCompressOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    MaskCompressOp::Adaptor adaptor(operands);
+    auto opType = adaptor.a().getType();
+
+    Value src;
+    if (op.src()) {
+      src = adaptor.src();
+    } else if (op.constant_src()) {
+      src = rewriter.create<ConstantOp>(op.getLoc(), opType,
+                                        op.constant_srcAttr());
+    } else {
+      Attribute zeroAttr = rewriter.getZeroAttr(opType);
+      src = rewriter.create<ConstantOp>(op->getLoc(), opType, zeroAttr);
+    }
+
+    rewriter.replaceOpWithNewOp<MaskCompressIntrOp>(op, opType, adaptor.a(),
+                                                    src, adaptor.k());
+
+    return success();
+  }
+};
+
+/// An entry associating the "main" AVX512 op with its instantiations for
+/// vectors of 32-bit and 64-bit elements.
+template <typename OpTy, typename Intr32OpTy, typename Intr64OpTy>
+struct RegEntry {
+  using MainOp = OpTy;
+  using Intr32Op = Intr32OpTy;
+  using Intr64Op = Intr64OpTy;
+};
+
+/// A container for op association entries facilitating the configuration of
+/// dialect conversion.
+template <typename... Args>
+struct RegistryImpl {
+  /// Registers the patterns specializing the "main" op to one of the
+  /// "intrinsic" ops depending on elemental type.
+  static void registerPatterns(LLVMTypeConverter &converter,
+                               OwningRewritePatternList &patterns) {
+    patterns
+        .insert<LowerToIntrinsic<typename Args::MainOp, typename Args::Intr32Op,
+                                 typename Args::Intr64Op>...>(converter);
+  }
+
+  /// Configures the conversion target to lower out "main" ops.
+  static void configureTarget(LLVMConversionTarget &target) {
+    target.addIllegalOp<typename Args::MainOp...>();
+    target.addLegalOp<typename Args::Intr32Op...>();
+    target.addLegalOp<typename Args::Intr64Op...>();
+  }
+};
+
+using Registry = RegistryImpl<
+    RegEntry<MaskRndScaleOp, MaskRndScalePSIntrOp, MaskRndScalePDIntrOp>,
+    RegEntry<MaskScaleFOp, MaskScaleFPSIntrOp, MaskScaleFPDIntrOp>,
+    RegEntry<Vp2IntersectOp, Vp2IntersectDIntrOp, Vp2IntersectQIntrOp>>;
+
+} // namespace
+
+/// Populate the given list with patterns that convert from AVX512 to LLVM.
+void mlir::populateAVX512LegalizeForLLVMExportPatterns(
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+  Registry::registerPatterns(converter, patterns);
+  patterns.insert<MaskCompressOpConversion>(converter);
+}
+
+void mlir::configureAVX512LegalizeForExportTarget(
+    LLVMConversionTarget &target) {
+  Registry::configureTarget(target);
+  target.addLegalOp<MaskCompressIntrOp>();
+  target.addIllegalOp<MaskCompressOp>();
+}

mlir/lib/Dialect/LLVMIR/CMakeLists.txt

add_mlir_dialect_library(MLIRLLVMIR
   LINK_LIBS PUBLIC
   MLIRCallInterfaces
   MLIRControlFlowInterfaces
   MLIRIR
   MLIRSideEffectInterfaces
   MLIRSupport
   )
 
-add_mlir_dialect_library(MLIRLLVMAVX512
-  IR/LLVMAVX512Dialect.cpp
-
-  ADDITIONAL_HEADER_DIRS
-  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
-
-  DEPENDS
-  MLIRLLVMAVX512IncGen
-  MLIRLLVMAVX512ConversionsIncGen
-  intrinsics_gen
-
-  LINK_COMPONENTS
-  AsmParser
-  Core
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRLLVMIR
-  MLIRSideEffectInterfaces
-  )
-
 add_mlir_dialect_library(MLIRLLVMArmSVE
   IR/LLVMArmSVEDialect.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
 
   DEPENDS
   MLIRLLVMArmSVEIncGen

mlir/lib/Dialect/LLVMIR/IR/LLVMAVX512Dialect.cpp

-//===- LLVMAVX512Dialect.cpp - MLIR LLVMAVX512 ops implementation ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the LLVMAVX512 dialect and its operations.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/IR/IntrinsicsX86.h"
-
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/TypeUtilities.h"
-
-using namespace mlir;
-
-void LLVM::LLVMAVX512Dialect::initialize() {
-  addOperations<
-#define GET_OP_LIST
-#include "mlir/Dialect/LLVMIR/LLVMAVX512.cpp.inc"
-      >();
-}
-
-#define GET_OP_CLASSES
-#include "mlir/Dialect/LLVMIR/LLVMAVX512.cpp.inc"

mlir/lib/Target/LLVMIR/CMakeLists.txt

add_mlir_translation_library(MLIRTargetLLVMIRExport
   MLIRTranslation
   )
 
 add_mlir_translation_library(MLIRToLLVMIRTranslationRegistration
   ConvertToLLVMIR.cpp
 
   LINK_LIBS PUBLIC
   MLIRArmNeonToLLVMIRTranslation
+  MLIRAVX512ToLLVMIRTranslation
   MLIRLLVMArmSVEToLLVMIRTranslation
-  MLIRLLVMAVX512ToLLVMIRTranslation
   MLIRLLVMToLLVMIRTranslation
   MLIRNVVMToLLVMIRTranslation
   MLIROpenMPToLLVMIRTranslation
   MLIRROCDLToLLVMIRTranslation
   )
 
 add_mlir_translation_library(MLIRTargetLLVMIRImport
   ConvertFromLLVMIR.cpp

mlir/lib/Target/LLVMIR/Dialect/AVX512/AVX512ToLLVMIRTranslation.cpp

-//===- LLVMAVX512ToLLVMIRTranslation.cpp - Translate LLVMAVX512 to LLVM IR-===//
+//===- AVX512ToLLVMIRTranslation.cpp - Translate AVX512 to LLVM IR---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements a translation between the MLIR LLVMAVX512 dialect and
+// This file implements a translation between the MLIR AVX512 dialect and
 // LLVM IR.
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Target/LLVMIR/Dialect/LLVMAVX512/LLVMAVX512ToLLVMIRTranslation.h"
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
+#include "mlir/Target/LLVMIR/Dialect/AVX512/AVX512ToLLVMIRTranslation.h"
+#include "mlir/Dialect/AVX512/AVX512Dialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsX86.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
 
 namespace {
 /// Implementation of the dialect interface that converts operations belonging
-/// to the LLVMAVX512 dialect to LLVM IR.
-class LLVMAVX512DialectLLVMIRTranslationInterface
+/// to the AVX512 dialect to LLVM IR.
+class AVX512DialectLLVMIRTranslationInterface
     : public LLVMTranslationDialectInterface {
 public:
   using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
 
   /// Translates the given operation to LLVM IR using the provided IR builder
   /// and saving the state in `moduleTranslation`.
   LogicalResult
   convertOperation(Operation *op, llvm::IRBuilderBase &builder,
                    LLVM::ModuleTranslation &moduleTranslation) const final {
     Operation &opInst = *op;
-#include "mlir/Dialect/LLVMIR/LLVMAVX512Conversions.inc"
+#include "mlir/Dialect/AVX512/AVX512Conversions.inc"
 
     return failure();
   }
 };
 } // end namespace
 
-void mlir::registerLLVMAVX512DialectTranslation(DialectRegistry &registry) {
-  registry.insert<LLVM::LLVMAVX512Dialect>();
-  registry.addDialectInterface<LLVM::LLVMAVX512Dialect,
-                               LLVMAVX512DialectLLVMIRTranslationInterface>();
+void mlir::registerAVX512DialectTranslation(DialectRegistry &registry) {
+  registry.insert<avx512::AVX512Dialect>();
+  registry.addDialectInterface<avx512::AVX512Dialect,
+                               AVX512DialectLLVMIRTranslationInterface>();
 }
 
-void mlir::registerLLVMAVX512DialectTranslation(MLIRContext &context) {
+void mlir::registerAVX512DialectTranslation(MLIRContext &context) {
   DialectRegistry registry;
-  registerLLVMAVX512DialectTranslation(registry);
+  registerAVX512DialectTranslation(registry);
   context.appendDialectRegistry(registry);
 }

mlir/lib/Target/LLVMIR/Dialect/AVX512/CMakeLists.txt

+add_mlir_translation_library(MLIRAVX512ToLLVMIRTranslation
+  AVX512ToLLVMIRTranslation.cpp
+
+  DEPENDS
+  MLIRAVX512ConversionsIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRAVX512
+  MLIRLLVMIR
+  MLIRSupport
+  MLIRTargetLLVMIRExport
+  )

mlir/lib/Target/LLVMIR/Dialect/CMakeLists.txt

 add_subdirectory(ArmNeon)
+add_subdirectory(AVX512)
 add_subdirectory(LLVMArmSVE)
-add_subdirectory(LLVMAVX512)
 add_subdirectory(LLVMIR)
 add_subdirectory(NVVM)
 add_subdirectory(OpenMP)
 add_subdirectory(ROCDL)

mlir/lib/Target/LLVMIR/Dialect/LLVMAVX512/CMakeLists.txt

-add_mlir_translation_library(MLIRLLVMAVX512ToLLVMIRTranslation
-  LLVMAVX512ToLLVMIRTranslation.cpp
-
-  DEPENDS
-  MLIRLLVMAVX512ConversionsIncGen
-
-  LINK_COMPONENTS
-  Core
-
-  LINK_LIBS PUBLIC
-  MLIRIR
-  MLIRLLVMAVX512
-  MLIRLLVMIR
-  MLIRSupport
-  MLIRTargetLLVMIRExport
-  )

mlir/test/Dialect/AVX512/legalize-for-llvm.mlir

 // RUN: mlir-opt %s -convert-vector-to-llvm="enable-avx512" | mlir-opt | FileCheck %s
 
 func @avx512_mask_rndscale(%a: vector<16xf32>, %b: vector<8xf64>, %i32: i32, %i16: i16, %i8: i8)
   -> (vector<16xf32>, vector<8xf64>, vector<16xf32>, vector<8xf64>)
 {
-  // CHECK: llvm_avx512.mask.rndscale.ps.512
+  // CHECK: avx512.intr.mask.rndscale.ps.512
   %0 = avx512.mask.rndscale %a, %i32, %a, %i16, %i32: vector<16xf32>
-  // CHECK: llvm_avx512.mask.rndscale.pd.512
+  // CHECK: avx512.intr.mask.rndscale.pd.512
   %1 = avx512.mask.rndscale %b, %i32, %b, %i8, %i32: vector<8xf64>
 
-  // CHECK: llvm_avx512.mask.scalef.ps.512
+  // CHECK: avx512.intr.mask.scalef.ps.512
   %2 = avx512.mask.scalef %a, %a, %a, %i16, %i32: vector<16xf32>
-  // CHECK: llvm_avx512.mask.scalef.pd.512
+  // CHECK: avx512.intr.mask.scalef.pd.512
   %3 = avx512.mask.scalef %b, %b, %b, %i8, %i32: vector<8xf64>
 
   // Keep results alive.
   return %0, %1, %2, %3 : vector<16xf32>, vector<8xf64>, vector<16xf32>, vector<8xf64>
 }
 
 func @avx512_mask_compress(%k1: vector<16xi1>, %a1: vector<16xf32>,
                            %k2: vector<8xi1>, %a2: vector<8xi64>)
   -> (vector<16xf32>, vector<16xf32>, vector<8xi64>)
 {
-  // CHECK: llvm_avx512.mask.compress
+  // CHECK: avx512.intr.mask.compress
   %0 = avx512.mask.compress %k1, %a1 : vector<16xf32>
-  // CHECK: llvm_avx512.mask.compress
+  // CHECK: avx512.intr.mask.compress
   %1 = avx512.mask.compress %k1, %a1 {constant_src = dense<5.0> : vector<16xf32>} : vector<16xf32>
-  // CHECK: llvm_avx512.mask.compress
+  // CHECK: avx512.intr.mask.compress
   %2 = avx512.mask.compress %k2, %a2, %a2 : vector<8xi64>, vector<8xi64>
   return %0, %1, %2 : vector<16xf32>, vector<16xf32>, vector<8xi64>
 }
 
 func @avx512_vp2intersect(%a: vector<16xi32>, %b: vector<8xi64>)
   -> (vector<16xi1>, vector<16xi1>, vector<8xi1>, vector<8xi1>)
 {
-  // CHECK: llvm_avx512.vp2intersect.d.512
+  // CHECK: avx512.intr.vp2intersect.d.512
   %0, %1 = avx512.vp2intersect %a, %a : vector<16xi32>
-  // CHECK: llvm_avx512.vp2intersect.q.512
+  // CHECK: avx512.intr.vp2intersect.q.512
   %2, %3 = avx512.vp2intersect %b, %b : vector<8xi64>
   return %0, %1, %2, %3 : vector<16xi1>, vector<16xi1>, vector<8xi1>, vector<8xi1>
 }

mlir/test/Target/LLVMIR/avx512.mlir

 // RUN: mlir-translate --mlir-to-llvmir %s | FileCheck %s
 
 // CHECK-LABEL: define <16 x float> @LLVM_x86_avx512_mask_ps_512
 llvm.func @LLVM_x86_avx512_mask_ps_512(%a: vector<16 x f32>,
                                        %c: i16)
   -> (vector<16 x f32>)
 {
   %b = llvm.mlir.constant(42 : i32) : i32
   // CHECK: call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float>
-  %0 = "llvm_avx512.mask.rndscale.ps.512"(%a, %b, %a, %c, %b) :
+  %0 = "avx512.intr.mask.rndscale.ps.512"(%a, %b, %a, %c, %b) :
     (vector<16 x f32>, i32, vector<16 x f32>, i16, i32) -> vector<16 x f32>
   // CHECK: call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float>
-  %1 = "llvm_avx512.mask.scalef.ps.512"(%a, %a, %a, %c, %b) :
+  %1 = "avx512.intr.mask.scalef.ps.512"(%a, %a, %a, %c, %b) :
     (vector<16 x f32>, vector<16 x f32>, vector<16 x f32>, i16, i32) -> vector<16 x f32>
   llvm.return %1: vector<16 x f32>
 }
 
 // CHECK-LABEL: define <8 x double> @LLVM_x86_avx512_mask_pd_512
 llvm.func @LLVM_x86_avx512_mask_pd_512(%a: vector<8xf64>,
                                        %c: i8)
   -> (vector<8xf64>)
 {
   %b = llvm.mlir.constant(42 : i32) : i32
   // CHECK: call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double>
-  %0 = "llvm_avx512.mask.rndscale.pd.512"(%a, %b, %a, %c, %b) :
+  %0 = "avx512.intr.mask.rndscale.pd.512"(%a, %b, %a, %c, %b) :
     (vector<8xf64>, i32, vector<8xf64>, i8, i32) -> vector<8xf64>
   // CHECK: call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double>
-  %1 = "llvm_avx512.mask.scalef.pd.512"(%a, %a, %a, %c, %b) :
+  %1 = "avx512.intr.mask.scalef.pd.512"(%a, %a, %a, %c, %b) :
     (vector<8xf64>, vector<8xf64>, vector<8xf64>, i8, i32) -> vector<8xf64>
   llvm.return %1: vector<8xf64>
 }
 
 // CHECK-LABEL: define <16 x float> @LLVM_x86_mask_compress
 llvm.func @LLVM_x86_mask_compress(%k: vector<16xi1>, %a: vector<16xf32>)
   -> vector<16xf32>
 {
   // CHECK: call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(
-  %0 = "llvm_avx512.mask.compress"(%a, %a, %k) :
+  %0 = "avx512.intr.mask.compress"(%a, %a, %k) :
     (vector<16xf32>, vector<16xf32>, vector<16xi1>) -> vector<16xf32>
   llvm.return %0 : vector<16xf32>
 }
 
 // CHECK-LABEL: define { <16 x i1>, <16 x i1> } @LLVM_x86_vp2intersect_d_512
 llvm.func @LLVM_x86_vp2intersect_d_512(%a: vector<16xi32>, %b: vector<16xi32>)
   -> !llvm.struct<(vector<16 x i1>, vector<16 x i1>)>
 {
   // CHECK: call { <16 x i1>, <16 x i1> } @llvm.x86.avx512.vp2intersect.d.512(<16 x i32>
-  %0 = "llvm_avx512.vp2intersect.d.512"(%a, %b) :
+  %0 = "avx512.intr.vp2intersect.d.512"(%a, %b) :
     (vector<16xi32>, vector<16xi32>) -> !llvm.struct<(vector<16 x i1>, vector<16 x i1>)>
   llvm.return %0 : !llvm.struct<(vector<16 x i1>, vector<16 x i1>)>
 }
 
 // CHECK-LABEL: define { <8 x i1>, <8 x i1> } @LLVM_x86_vp2intersect_q_512
 llvm.func @LLVM_x86_vp2intersect_q_512(%a: vector<8xi64>, %b: vector<8xi64>)
   -> !llvm.struct<(vector<8 x i1>, vector<8 x i1>)>
 {
   // CHECK: call { <8 x i1>, <8 x i1> } @llvm.x86.avx512.vp2intersect.q.512(<8 x i64>
-  %0 = "llvm_avx512.vp2intersect.q.512"(%a, %b) :
+  %0 = "avx512.intr.vp2intersect.q.512"(%a, %b) :
     (vector<8xi64>, vector<8xi64>) -> !llvm.struct<(vector<8 x i1>, vector<8 x i1>)>
   llvm.return %0 : !llvm.struct<(vector<8 x i1>, vector<8 x i1>)>
 }

mlir/test/mlir-opt/commandline.mlir

 // RUN: mlir-opt --show-dialects | FileCheck %s
 // CHECK: Available Dialects:
 // CHECK-NEXT: acc
 // CHECK-NEXT: affine
 // CHECK-NEXT: arm_neon
 // CHECK-NEXT: arm_sve
 // CHECK-NEXT: async
 // CHECK-NEXT: avx512
 // CHECK-NEXT: complex
 // CHECK-NEXT: gpu
 // CHECK-NEXT: linalg
 // CHECK-NEXT: llvm
 // CHECK-NEXT: llvm_arm_sve
-// CHECK-NEXT: llvm_avx512
 // CHECK-NEXT: math
 // CHECK-NEXT: nvvm
 // CHECK-NEXT: omp
 // CHECK-NEXT: pdl
 // CHECK-NEXT: pdl_interp
 // CHECK-NEXT: quant
 // CHECK-NEXT: rocdl
 // CHECK-NEXT: scf
 // CHECK-NEXT: sdbm
 // CHECK-NEXT: shape
 // CHECK-NEXT: spv
 // CHECK-NEXT: std
 // CHECK-NEXT: tensor
 // CHECK-NEXT: test
 // CHECK-NEXT: tosa
 // CHECK-NEXT: vector