[CodeGen][AArch64][SVE] Fold [rdffr, ptest] => rdffrs for RDFFR_P

[peterwaller-arm]

Summary

Follow up to D101357 / 3fa6510f6.

Use flags setting rdffrs instead of rddfr.

RDFFR_P has to be handled differently from the other optimizations in
optimizePTestInstr: It requires introducing an additional operand, since
there is no RDFFRS_P, only RDFFRS_PP. In order to use a flag setting
version, we need to move the mask operand from the PTest into the
flags-setting instruction.

The mask therefore needs to be defined before the RDFFR. If it is not,
bail out for now, which seems to be sufficient for typical code where
this is encountered. This could be extended to move the definition of
the mask up so long as there are no other users.

[peterwaller-arm]

Run clang-format.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

1372

This is only equivalent when p0 is all true or when the result of the rdffr is not used (i.e. dead).

[peterwaller-arm]

I plan to follow this up early next week with an alternative approach.

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

static bool areCFlagsAccessedBetweenInstrs(
   return false;
 }
 
 /// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
 /// operation which could set the flags in an identical manner
 bool AArch64InstrInfo::optimizePTestInstr(
     MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
     const MachineRegisterInfo *MRI) const {
+  const TargetRegisterInfo *TRI = &getRegisterInfo();
+
   auto *Mask = MRI->getUniqueVRegDef(MaskReg);
   auto *Pred = MRI->getUniqueVRegDef(PredReg);
   auto NewOp = Pred->getOpcode();
   bool OpChanged = false;
 
   unsigned MaskOpcode = Mask->getOpcode();
   unsigned PredOpcode = Pred->getOpcode();
   bool PredIsPTestLike = isPTestLikeOpcode(PredOpcode);
case AArch64::BRKN_PPzP: {
       auto *PredMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
       if (Mask != PredMask)
         return false;
 
       NewOp = AArch64::BRKNS_PPzP;
       OpChanged = true;
       break;
     }
+    case AArch64::RDFFR_P: {
+      // rdffr   p1.b <--- Optimize this to rdffrs p1.b, p0/z
+      // <between instructions no defs of Mask nor use/def of Flags>
+      // ptest   Mask=p0, Pred=p1.b
+      if (areCFlagsAccessedBetweenInstrs(Pred, PTest, TRI))
+        return false;
+
+      // If Mask is def'd between the two, the def needs moving up.
+      // Bail out for now.
+      if (any_of(instructionsWithoutDebug(std::next(Pred->getIterator()),
+                                          PTest->getIterator()),
+                 [TRI, MaskReg](const MachineInstr &I) {
+                   return I.definesRegister(MaskReg, TRI);
+                 }))
+        return false;
+
+      BuildMI(*Pred->getParent(), Pred, Pred->getDebugLoc(),
+              get(AArch64::RDFFRS_PPz))
+          .addDef(PredReg)
+          .addUse(MaskReg);
+      PTest->eraseFromParent();
+      Pred->eraseFromParent();
+      return true;
+    }
     case AArch64::RDFFR_PPz: {
       // rdffr   p1.b, PredMask=p0/z <--- Definition of Pred
       // ptest   Mask=p0, Pred=p1.b  <--- If equal masks, remove this and use
       //                                  `rdffrs p1.b, p0/z` above.
       auto *PredMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
       if (Mask != PredMask)
         return false;
 
       NewOp = AArch64::RDFFRS_PPz;
       OpChanged = true;
       break;
     }
     default:
       // Bail out if we don't recognize the input
       return false;
     }
   }
 
-  const TargetRegisterInfo *TRI = &getRegisterInfo();
-
   // If another instruction between Pred and PTest accesses flags, don't remove
   // the ptest or update the earlier instruction to modify them.
   if (areCFlagsAccessedBetweenInstrs(Pred, PTest, TRI))
     return false;
 
   // If we pass all the checks, it's safe to remove the PTEST and use the flags
   // as they are prior to PTEST. Sometimes this requires the tested PTEST
   // operand to be replaced with an equivalent instruction that also sets the

llvm/test/CodeGen/AArch64/sve-ptest-removal-rdffr.mir

bb.0:
     %2:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
     PTEST_PP killed %0:ppr_3b, killed %1:ppr_3b, implicit-def $nzcv
 
     ; Consume nzcv
     %3:gpr32 = COPY $wzr
     %4:gpr32 = CSINCWr killed %3, $wzr, 0, implicit $nzcv
     $w0 = ORRWrs %4, %2, 1
     RET_ReallyLR implicit $w0
+...
+---
+# CHECK-LABEL: name:{{\s*}} substitute_rdffr_p_with_rdffrs_pp
+name:                       substitute_rdffr_p_with_rdffrs_pp
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $ffr, $p0
+    %0:ppr_3b = COPY $p0
+
+    ; CHECK: RDFFRS_PP
+    ; CHECK-NOT: PTEST
+    %1:ppr_3b = RDFFR_P
+    PTEST_PP killed %0:ppr_3b, killed %1:ppr_3b, implicit-def $nzcv
+
+    ; Consume nzcv
+    %2:gpr32 = COPY $wzr
+    %3:gpr32 = CSINCWr killed %2, $wzr, 0, implicit $nzcv
+    $w0 = COPY %3
+    RET_ReallyLR implicit $w0
+
+...
+---
+# CHECK-LABEL: name:{{\s*}} fail_to_substitute_rdffr_p_with_rdffrs_pp_def_between
+name:                       fail_to_substitute_rdffr_p_with_rdffrs_pp_def_between
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $ffr, $p0
+
+    ; CHECK-NOT: RDFFRS_PP
+    ; CHECK: PTEST
+    %0:ppr_3b = RDFFR_P
+    %1:ppr_3b = COPY $p0
+    PTEST_PP killed %1:ppr_3b, killed %0:ppr_3b, implicit-def $nzcv
+
+    ; Consume nzcv
+    %2:gpr32 = COPY $wzr
+    %3:gpr32 = CSINCWr killed %2, $wzr, 0, implicit $nzcv
+    $w0 = COPY %3
+    RET_ReallyLR implicit $w0