diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -663,11 +663,17 @@ defm vredmaxu : RISCVReduction; defm vredmax : RISCVReduction; + defm vwredsumu : RISCVReduction; + defm vwredsum : RISCVReduction; + defm vfredosum : RISCVReduction; defm vfredsum : RISCVReduction; defm vfredmin : RISCVReduction; defm vfredmax : RISCVReduction; + defm vfwredsum : RISCVReduction; + defm vfwredosum : RISCVReduction; + def int_riscv_vmand: RISCVBinaryAAANoMask; def int_riscv_vmnand: RISCVBinaryAAANoMask; def int_riscv_vmandnot: RISCVBinaryAAANoMask; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1799,6 +1799,22 @@ } } +multiclass VPatReductionW_VS { + foreach vti = !if(IsFloat, AllFloatVectors, AllIntegerVectors) in + { + defvar wtiSEW = !mul(vti.SEW, 2); + if !le(wtiSEW, 64) then { + defvar wtiM1 = !cast(!if(IsFloat, "VF", "VI") # wtiSEW # "M1"); + defm : VPatTernary; + } + } +} + //===----------------------------------------------------------------------===// // Pseudo instructions and patterns. //===----------------------------------------------------------------------===// @@ -2138,6 +2154,12 @@ defm PseudoVREDMIN : VPseudoReductionV_VS; defm PseudoVREDMAXU : VPseudoReductionV_VS; defm PseudoVREDMAX : VPseudoReductionV_VS; + +//===----------------------------------------------------------------------===// +// 15.2. Vector Widening Integer Reduction Instructions +//===----------------------------------------------------------------------===// +defm PseudoVWREDSUMU : VPseudoReductionV_VS; +defm PseudoVWREDSUM : VPseudoReductionV_VS; } // Predicates = [HasStdExtV] let Predicates = [HasStdExtV, HasStdExtF] in { @@ -2148,6 +2170,12 @@ defm PseudoVFREDSUM : VPseudoReductionV_VS; defm PseudoVFREDMIN : VPseudoReductionV_VS; defm PseudoVFREDMAX : VPseudoReductionV_VS; + +//===----------------------------------------------------------------------===// +// 15.4. Vector Widening Floating-Point Reduction Instructions +//===----------------------------------------------------------------------===// +defm PseudoVFWREDSUM : VPseudoReductionV_VS; +defm PseudoVFWREDOSUM : VPseudoReductionV_VS; } // Predicates = [HasStdExtV, HasStdExtF] //===----------------------------------------------------------------------===// @@ -2630,6 +2658,12 @@ defm "" : VPatReductionV_VS<"int_riscv_vredmin", "PseudoVREDMIN">; defm "" : VPatReductionV_VS<"int_riscv_vredmaxu", "PseudoVREDMAXU">; defm "" : VPatReductionV_VS<"int_riscv_vredmax", "PseudoVREDMAX">; + +//===----------------------------------------------------------------------===// +// 15.2. Vector Widening Integer Reduction Instructions +//===----------------------------------------------------------------------===// +defm "" : VPatReductionW_VS<"int_riscv_vwredsumu", "PseudoVWREDSUMU">; +defm "" : VPatReductionW_VS<"int_riscv_vwredsum", "PseudoVWREDSUM">; } // Predicates = [HasStdExtV] let Predicates = [HasStdExtV, HasStdExtF] in { @@ -2640,6 +2674,12 @@ defm "" : VPatReductionV_VS<"int_riscv_vfredsum", "PseudoVFREDSUM", /*IsFloat=*/1>; defm "" : VPatReductionV_VS<"int_riscv_vfredmin", "PseudoVFREDMIN", /*IsFloat=*/1>; defm "" : VPatReductionV_VS<"int_riscv_vfredmax", "PseudoVFREDMAX", /*IsFloat=*/1>; + +//===----------------------------------------------------------------------===// +// 15.4. Vector Widening Floating-Point Reduction Instructions +//===----------------------------------------------------------------------===// +defm "" : VPatReductionW_VS<"int_riscv_vfwredsum", "PseudoVFWREDSUM", /*IsFloat=*/1>; +defm "" : VPatReductionW_VS<"int_riscv_vfwredosum", "PseudoVFWREDOSUM", /*IsFloat=*/1>; } // Predicates = [HasStdExtV, HasStdExtF] //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredosum-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredosum-rv32.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/vfwredosum-rv32.ll @@ -0,0 +1,43 @@ +; RUN: llc -mtriple=riscv32 -mattr=+experimental-v,+f,+experimental-zfh -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s +declare @llvm.riscv.vfwredosum.nxv2f32.nxv32f16( + , + , + , + i32); + +define @intrinsic_vfwredosum_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, i32 %3) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} + %a = call @llvm.riscv.vfwredosum.nxv2f32.nxv32f16( + %0, + %1, + %2, + i32 %3) + + ret %a +} + +declare @llvm.riscv.vfwredosum.mask.nxv2f32.nxv32f16.nxv32i1( + , + , + , + , + i32); + +define @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, %3, i32 %4) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t + %a = call @llvm.riscv.vfwredosum.mask.nxv2f32.nxv32f16.nxv32i1( + %0, + %1, + %2, + %3, + i32 %4) + + ret %a +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredosum-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredosum-rv64.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/vfwredosum-rv64.ll @@ -0,0 +1,85 @@ +; RUN: llc -mtriple=riscv64 -mattr=+experimental-v,+d,+experimental-zfh -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s +declare @llvm.riscv.vfwredosum.nxv2f32.nxv32f16( + , + , + , + i64); + +define @intrinsic_vfwredosum_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, i64 %3) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} + %a = call @llvm.riscv.vfwredosum.nxv2f32.nxv32f16( + %0, + %1, + %2, + i64 %3) + + ret %a +} + +declare @llvm.riscv.vfwredosum.mask.nxv2f32.nxv32f16.nxv32i1( + , + , + , + , + i64); + +define @intrinsic_vfwredosum_mask_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, %3, i64 %4) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t + %a = call @llvm.riscv.vfwredosum.mask.nxv2f32.nxv32f16.nxv32i1( + %0, + %1, + %2, + %3, + i64 %4) + + ret %a +} + +declare @llvm.riscv.vfwredosum.nxv1f64.nxv16f32( + , + , + , + i64); + +define @intrinsic_vfwredosum_vs_nxv1f64_nxv16f32_nxv1f64( %0, %1, %2, i64 %3) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredosum_vs_nxv1f64_nxv16f32_nxv1f64 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e32,m8,ta,mu +; CHECK: vfwredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} + %a = call @llvm.riscv.vfwredosum.nxv1f64.nxv16f32( + %0, + %1, + %2, + i64 %3) + + ret %a +} + +declare @llvm.riscv.vfwredosum.mask.nxv1f64.nxv16f32.nxv16i1( + , + , + , + , + i64); + +define @intrinsic_vfwredosum_mask_vs_nxv1f64_nxv16f32_nxv1f64( %0, %1, %2, %3, i64 %4) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredosum_mask_vs_nxv1f64_nxv16f32_nxv1f64 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e32,m8,ta,mu +; CHECK: vfwredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t + %a = call @llvm.riscv.vfwredosum.mask.nxv1f64.nxv16f32.nxv16i1( + %0, + %1, + %2, + %3, + i64 %4) + + ret %a +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredsum-rv32.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredsum-rv32.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/vfwredsum-rv32.ll @@ -0,0 +1,43 @@ +; RUN: llc -mtriple=riscv32 -mattr=+experimental-v,+f,+experimental-zfh -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s +declare @llvm.riscv.vfwredsum.nxv2f32.nxv32f16( + , + , + , + i32); + +define @intrinsic_vfwredsum_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, i32 %3) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredsum_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} + %a = call @llvm.riscv.vfwredsum.nxv2f32.nxv32f16( + %0, + %1, + %2, + i32 %3) + + ret %a +} + +declare @llvm.riscv.vfwredsum.mask.nxv2f32.nxv32f16.nxv32i1( + , + , + , + , + i32); + +define @intrinsic_vfwredsum_mask_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, %3, i32 %4) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredsum_mask_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t + %a = call @llvm.riscv.vfwredsum.mask.nxv2f32.nxv32f16.nxv32i1( + %0, + %1, + %2, + %3, + i32 %4) + + ret %a +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vfwredsum-rv64.ll b/llvm/test/CodeGen/RISCV/rvv/vfwredsum-rv64.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/vfwredsum-rv64.ll @@ -0,0 +1,85 @@ +; RUN: llc -mtriple=riscv64 -mattr=+experimental-v,+d,+experimental-zfh -verify-machineinstrs \ +; RUN: --riscv-no-aliases < %s | FileCheck %s +declare @llvm.riscv.vfwredsum.nxv2f32.nxv32f16( + , + , + , + i64); + +define @intrinsic_vfwredsum_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, i64 %3) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredsum_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} + %a = call @llvm.riscv.vfwredsum.nxv2f32.nxv32f16( + %0, + %1, + %2, + i64 %3) + + ret %a +} + +declare @llvm.riscv.vfwredsum.mask.nxv2f32.nxv32f16.nxv32i1( + , + , + , + , + i64); + +define @intrinsic_vfwredsum_mask_vs_nxv2f32_nxv32f16_nxv2f32( %0, %1, %2, %3, i64 %4) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredsum_mask_vs_nxv2f32_nxv32f16_nxv2f32 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e16,m8,ta,mu +; CHECK: vfwredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t + %a = call @llvm.riscv.vfwredsum.mask.nxv2f32.nxv32f16.nxv32i1( + %0, + %1, + %2, + %3, + i64 %4) + + ret %a +} + +declare @llvm.riscv.vfwredsum.nxv1f64.nxv16f32( + , + , + , + i64); + +define @intrinsic_vfwredsum_vs_nxv1f64_nxv16f32_nxv1f64( %0, %1, %2, i64 %3) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredsum_vs_nxv1f64_nxv16f32_nxv1f64 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e32,m8,ta,mu +; CHECK: vfwredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} + %a = call @llvm.riscv.vfwredsum.nxv1f64.nxv16f32( + %0, + %1, + %2, + i64 %3) + + ret %a +} + +declare @llvm.riscv.vfwredsum.mask.nxv1f64.nxv16f32.nxv16i1( + , + , + , + , + i64); + +define @intrinsic_vfwredsum_mask_vs_nxv1f64_nxv16f32_nxv1f64( %0, %1, %2, %3, i64 %4) nounwind { +entry: +; CHECK-LABEL: intrinsic_vfwredsum_mask_vs_nxv1f64_nxv16f32_nxv1f64 +; CHECK: vsetvli {{.*}}, {{a[0-9]+}}, e32,m8,ta,mu +; CHECK: vfwredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t + %a = call @llvm.riscv.vfwredsum.mask.nxv1f64.nxv16f32.nxv16i1( + %0, + %1, + %2, + %3, + i64 %4) + + ret %a +}