[CodeGen][AArch64][SVE] Canonicalize intrinsic rdffr{ => _z}

[peterwaller-arm]

Summary

Follow up to D101357 / 3fa6510f6.
Supersedes D102330.

Goal: Use flags setting rdffrs instead of rdffr + ptest.

Problem: RDFFR_P doesn't have have a flags setting equivalent.

Solution: in instcombine, canonicalize to RDFFR_PP at the IR level, and
rely on RDFFR_PP+PTEST => RDFFRS_PP optimization in
AArch64InstrInfo::optimizePTestInstr.

While here:

• Test that rdffr.z+ptest generates a rdffrs.
• Use update_{test,llc}_checks.py on the tests.
• Use sve attribute on functions.

[david-arm]

llvm/test/CodeGen/AArch64/sve-intrinsics-rdffr-predication.ll

1 ↗

(On Diff #345866)

Hi @peterwaller-arm, I think this test should probably live in Transforms/InstCombine/AArch64

[peterwaller-arm]

• Move test per review comment.
• Update ACLE test.
• Rebase.

[bsmith]

LGTM

clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_rdffr.c

 // REQUIRES: aarch64-registered-target
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s
 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -o - %s >/dev/null
 #include <arm_sve.h>
 
 svbool_t test_svrdffr()
 {
   // CHECK-LABEL: test_svrdffr
-  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+  // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1>
+  // CHECK-NOT: rdffr
   // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
   return svrdffr();
 }
 
 svbool_t test_svrdffr_z(svbool_t pg)
 {
   // CHECK-LABEL: test_svrdffr_z
   // CHECK: %[[INTRINSIC:.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> %pg)
   // CHECK: ret <vscale x 16 x i1> %[[INTRINSIC]]
   return svrdffr_z(pg);
 }

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

static Optional<Instruction *> instCombineSVELast(InstCombiner &IC,
   // The intrinsic is extracting a fixed lane so use an extract instead.
   auto *IdxTy = Type::getInt64Ty(II.getContext());
   auto *Extract = ExtractElementInst::Create(Vec, ConstantInt::get(IdxTy, Idx));
   Extract->insertBefore(&II);
   Extract->takeName(&II);
   return IC.replaceInstUsesWith(II, Extract);
 }
 
+static Optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
+                                                IntrinsicInst &II) {
+  LLVMContext &Ctx = II.getContext();
+  IRBuilder<> Builder(Ctx);
+  Builder.SetInsertPoint(&II);
+  // Replace rdffr with predicated rdffr.z intrinsic, so that optimizePTestInstr
+  // can work with RDFFR_PP for ptest elimination.
+  auto *AllPat =
+      ConstantInt::get(Type::getInt32Ty(Ctx), AArch64SVEPredPattern::all);
+  auto *PTrue = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue,
+                                        {II.getType()}, {AllPat});
+  auto *RDFFR =
+      Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue});
+  RDFFR->takeName(&II);
+  return IC.replaceInstUsesWith(II, RDFFR);
+}
+
 Optional<Instruction *>
 AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
                                      IntrinsicInst &II) const {
   Intrinsic::ID IID = II.getIntrinsicID();
   switch (IID) {
   default:
     break;
   case Intrinsic::aarch64_sve_convert_from_svbool:
     return instCombineConvertFromSVBool(IC, II);
   case Intrinsic::aarch64_sve_dup:
     return instCombineSVEDup(IC, II);
+  case Intrinsic::aarch64_sve_rdffr:
+    return instCombineRDFFR(IC, II);
   case Intrinsic::aarch64_sve_lasta:
   case Intrinsic::aarch64_sve_lastb:
     return instCombineSVELast(IC, II);
   }
 
   return None;
 }
 

llvm/test/CodeGen/AArch64/sve-intrinsics-ffr-manipulation.ll

-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
 
 ;
 ; RDFFR
 ;
 
-define <vscale x 16 x i1> @rdffr() {
+define <vscale x 16 x i1> @rdffr() #0 {
 ; CHECK-LABEL: rdffr:
-; CHECK: rdffr p0.b
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdffr p0.b
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
   ret <vscale x 16 x i1> %out
 }
 
-define <vscale x 16 x i1> @rdffr_z(<vscale x 16 x i1> %pg) {
+define <vscale x 16 x i1> @rdffr_z(<vscale x 16 x i1> %pg) #0 {
 ; CHECK-LABEL: rdffr_z:
-; CHECK: rdffr p0.b, p0/z
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdffr p0.b, p0/z
+; CHECK-NEXT:    ret
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> %pg)
   ret <vscale x 16 x i1> %out
 }
 
+; Test that rdffr.z followed by ptest optimizes to flags-setting rdffrs.
+define i1 @rdffr_z_ptest(<vscale x 16 x i1> %pg) #0 {
+; CHECK-LABEL: rdffr_z_ptest:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rdffrs p0.b, p0/z
+; CHECK-NEXT:    cset w0, ne
+; CHECK-NEXT:    ret
+  %rdffr = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> %pg)
+  %out = call i1 @llvm.aarch64.sve.ptest.any.nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %rdffr)
+  ret i1 %out
+}
+
 ;
 ; SETFFR
 ;
 
-define void @set_ffr() {
+define void @set_ffr() #0 {
 ; CHECK-LABEL: set_ffr:
-; CHECK: setffr
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    setffr
+; CHECK-NEXT:    ret
   call void @llvm.aarch64.sve.setffr()
   ret void
 }
 
 ;
 ; WRFFR
 ;
 
-define void @wrffr(<vscale x 16 x i1> %a) {
+define void @wrffr(<vscale x 16 x i1> %a) #0 {
 ; CHECK-LABEL: wrffr:
-; CHECK: wrffr p0.b
-; CHECK-NEXT: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    wrffr p0.b
+; CHECK-NEXT:    ret
   call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> %a)
   ret void
 }
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
 declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1>)
 declare void @llvm.aarch64.sve.setffr()
 declare void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1>)
+
+declare i1 @llvm.aarch64.sve.ptest.any.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>)
+
+attributes #0 = { "target-features"="+sve" }

llvm/test/Transforms/InstCombine/AArch64/sve-intrinsics-rdffr-predication.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Test that rdffr is substituted with predicated form which enables ptest optimization later.
+define <vscale x 16 x i1> @predicate_rdffr() #0 {
+; CHECK-LABEL: @predicate_rdffr(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)
+; CHECK-NEXT:    [[OUT:%.*]] = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 16 x i1> [[OUT]]
+;
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+  ret <vscale x 16 x i1> %out
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+
+attributes #0 = { "target-features"="+sve" }