AArch64: Add FuseCryptoEOR fusion rules

[MatzeB]

Summary

There's some additional macro fusion opportunities on newer apple CPUs.

[javed.absar]

lib/Target/AArch64/AArch64.td

159

Would it be better to give a more operational name (e.g. FuseAESCrypto?) along with a comment if it applies to only certain configs. Just a thought, I am ok with as it is as well.

[evandro]

lib/Target/AArch64/AArch64.td

159

IIUC, this new feature is an extension of FeatureFuseAES. Therefore, I think that it'd be better if it were just about the new fusion, namely of PMULL and EOR.

351

Thereby retaining FeatureFuseAES here and adding the new feature.

lib/Target/AArch64/AArch64MacroFusion.cpp

128

The first two cases repeat isAESPair(), so methinks that it'd be better if it'd check only the new pair, PMULL AND EOR.

[MatzeB]

Address review feedback.

[MatzeB]

lib/Target/AArch64/AArch64.td

159

Yeah we already had FuseAES and I was just implementing another variation of it, hence my original try with "Apple" in the name. But indeed describing it more function like "CryptoEOR" turned out to look better.

351

ok, reused FeatureFuseAES this time and called the new thing FuseCryptoEOR.

[evandro]

Me likes it.

[MatzeB]

Landed in rL342590

lib/Target/AArch64/AArch64.td

 def FeatureFuseAddress : SubtargetFeature<
     "fuse-address", "HasFuseAddress", "true",
     "CPU fuses address generation and memory operations">;
 
 def FeatureFuseAES : SubtargetFeature<
     "fuse-aes", "HasFuseAES", "true",
     "CPU fuses AES crypto operations">;
 
+def FeatureFuseCryptoEOR : SubtargetFeature<
+    "fuse-crypto-eor", "HasFuseCryptoEOR", "true",
+    "CPU fuses AES/PMULL and EOR operations">;
+
 def FeatureFuseCCSelect : SubtargetFeature<
     "fuse-csel", "HasFuseCCSelect", "true",
     "CPU fuses conditional select operations">;
 
 def FeatureFuseLiterals : SubtargetFeature<
     "fuse-literals", "HasFuseLiterals", "true",
     "CPU fuses literal generation operations">;
 
def ProcCyclone : SubtargetFeature<"cyclone", "ARMProcFamily", "Cyclone",
                                    "Cyclone", [
                                    FeatureAlternateSExtLoadCVTF32Pattern,
                                    FeatureArithmeticBccFusion,
                                    FeatureArithmeticCbzFusion,
                                    FeatureCrypto,
                                    FeatureDisableLatencySchedHeuristic,
                                    FeatureFPARMv8,
                                    FeatureFuseAES,
+                                   FeatureFuseCryptoEOR,
                                    FeatureNEON,
                                    FeaturePerfMon,
                                    FeatureZCRegMove,
                                    FeatureZCZeroing,
                                    FeatureZCZeroingFPWorkaround
                                    ]>;
 
 def ProcExynosM1 : SubtargetFeature<"exynosm1", "ARMProcFamily", "ExynosM1",
                                     "Samsung Exynos-M1 processors",

lib/Target/AArch64/AArch64MacroFusion.cpp

else if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
             FirstOpcode == AArch64::AESDrr) &&
            (SecondOpcode == AArch64::AESIMCrr ||
             SecondOpcode == AArch64::AESIMCrrTied))
     return true;
 
   return false;
 }
 
+/// AESE/AESD/PMULL + EOR.
+static bool isCryptoEORPair(unsigned FirstOpcode, unsigned SecondOpcode) {
+  if (SecondOpcode != AArch64::EORv16i8)
+    return false;
+
+  switch (FirstOpcode) {
+  case AArch64::INSTRUCTION_LIST_END:
+  case AArch64::AESErr:
+  case AArch64::AESDrr:
+  case AArch64::PMULLv16i8:
+  case AArch64::PMULLv8i8:
+  case AArch64::PMULLv1i64:
+  case AArch64::PMULLv2i64:
+    return true;
+  }
+  return false;
+}
+
 /// Literal generation.
 static bool isLiteralsPair(unsigned FirstOpcode, unsigned SecondOpcode,
                            const MachineInstr *FirstMI,
                            const MachineInstr &SecondMI) {
   // PC relative address.
   if ((FirstOpcode == AArch64::INSTRUCTION_LIST_END ||
        FirstOpcode == AArch64::ADRP) &&
       SecondOpcode == AArch64::ADDXri)
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
   if (ST.hasArithmeticBccFusion() &&
       isArithmeticBccPair(FirstOpc, SecondOpc, FirstMI))
     return true;
   if (ST.hasArithmeticCbzFusion() &&
       isArithmeticCbzPair(FirstOpc, SecondOpc, FirstMI))
     return true;
   if (ST.hasFuseAES() && isAESPair(FirstOpc, SecondOpc))
     return true;
+  if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstOpc, SecondOpc))
+    return true;
   if (ST.hasFuseLiterals() &&
       isLiteralsPair(FirstOpc, SecondOpc, FirstMI, SecondMI))
     return true;
   if (ST.hasFuseAddress() && isAddressLdStPair(FirstOpc, SecondOpc, SecondMI))
     return true;
   if (ST.hasFuseCCSelect() && isCCSelectPair(FirstOpc, SecondOpc, FirstMI))
     return true;
 

lib/Target/AArch64/AArch64Subtarget.h

protected:
   bool Misaligned128StoreIsSlow = false;
   bool Paired128IsSlow = false;
   bool STRQroIsSlow = false;
   bool UseAlternateSExtLoadCVTF32Pattern = false;
   bool HasArithmeticBccFusion = false;
   bool HasArithmeticCbzFusion = false;
   bool HasFuseAddress = false;
   bool HasFuseAES = false;
+  bool HasFuseCryptoEOR = false;
   bool HasFuseCCSelect = false;
   bool HasFuseLiterals = false;
   bool DisableLatencySchedHeuristic = false;
   bool UseRSqrt = false;
   uint8_t MaxInterleaveFactor = 2;
   uint8_t VectorInsertExtractBaseCost = 3;
   uint16_t CacheLineSize = 0;
   uint16_t PrefetchDistance = 0;
public:
   bool isSTRQroSlow() const { return STRQroIsSlow; }
   bool useAlternateSExtLoadCVTF32Pattern() const {
     return UseAlternateSExtLoadCVTF32Pattern;
   }
   bool hasArithmeticBccFusion() const { return HasArithmeticBccFusion; }
   bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; }
   bool hasFuseAddress() const { return HasFuseAddress; }
   bool hasFuseAES() const { return HasFuseAES; }
+  bool hasFuseCryptoEOR() const { return HasFuseCryptoEOR; }
   bool hasFuseCCSelect() const { return HasFuseCCSelect; }
   bool hasFuseLiterals() const { return HasFuseLiterals; }
 
   /// Return true if the CPU supports any kind of instruction fusion.
   bool hasFusion() const {
     return hasArithmeticBccFusion() || hasArithmeticCbzFusion() ||
            hasFuseAES() || hasFuseCCSelect() || hasFuseLiterals();
   }

test/CodeGen/AArch64/misched-fusion-crypto-eor.mir

+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-- -mattr=-fuse-aes,+crypto -misched-print-dags 2>&1 | FileCheck %s --check-prefixes=CHECK,NOFUSE
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-- -mattr=+fuse-aes,+crypto -misched-print-dags 2>&1 | FileCheck %s --check-prefixes=CHECK,FUSEAES
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-- -mattr=+fuse-aes,+fuse-crpto-eor-crypto,+crypto -misched-print-dags 2>&1 | FileCheck %s --check-prefixes=CHECK,FUSEAES,FUSECRYPTO
+# REQUIRES: asserts
+
+name: func
+body: |
+  bb.0:
+    ; CHECK: SU(0): %0:fpr128 = AESErr undef $q0, undef $q1
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES:    SU(1): Ord  Latency=0 Cluster
+    ; CHECK: SU(1): %1:fpr128 = AESMCrrTied %0:fpr128
+    %0:fpr128 = AESErr undef $q0, undef $q1
+    %1:fpr128 = AESMCrrTied %0
+
+    ; CHECK: SU(2): %2:fpr128 = AESErr undef $q2, undef $q3
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES:    SU(3): Ord  Latency=0 Cluster
+    ; CHECK: SU(3): dead %3:fpr128 = AESMCrr %2:fpr128
+    %2:fpr128 = AESErr undef $q2, undef $q3
+    %3:fpr128 = AESMCrr %2
+
+    ; CHECK: SU(4): %4:fpr128 = AESErr %1:fpr128, undef $q4
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES-NOT: SU({{.*}}): Ord
+    ; FUSECRYPTO: SU(5): Ord  Latency=0 Cluster
+    ; CHECK: SU(5): dead %5:fpr128 = EORv16i8 %4:fpr128, undef $q5
+    %4:fpr128 = AESErr %1, undef $q4
+    %5:fpr128 = EORv16i8 %4, undef $q5
+
+    ; CHECK: SU(6): %6:fpr128 = AESDrr undef $q0, undef $q1
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES:    SU(7): Ord  Latency=0 Cluster
+    ; CHECK: SU(7): %7:fpr128 = AESIMCrrTied %6:fpr128
+    %6:fpr128 = AESDrr undef $q0, undef $q1
+    %7:fpr128 = AESIMCrrTied %6
+
+    ; CHECK: SU(8): %8:fpr128 = AESDrr undef $q2, undef $q3
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES:    SU(9): Ord  Latency=0 Cluster
+    ; CHECK: SU(9): dead %9:fpr128 = AESIMCrr %8:fpr128
+    %8:fpr128 = AESDrr undef $q2, undef $q3
+    %9:fpr128 = AESIMCrr %8
+
+    ; CHECK: SU(10): %10:fpr128 = AESDrr %7:fpr128, undef $q0
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES-NOT: SU({{.*}}): Ord
+    ; FUSECRYPTO: SU(11): Ord  Latency=0 Cluster
+    ; CHECK: SU(11): dead %11:fpr128 = EORv16i8 %10:fpr128, undef $q1
+    %10:fpr128 = AESDrr %7, undef $q0
+    %11:fpr128 = EORv16i8 %10, undef $q1
+
+    ; CHECK: SU(12): %12:fpr128 = PMULLv16i8 undef $q0, undef $q1
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES-NOT: SU({{.*}}): Ord
+    ; FUSECRYPTO: SU(13): Ord  Latency=0 Cluster
+    ; CHECK: SU(13): dead %13:fpr128 = EORv16i8 %12:fpr128, undef $q2
+    %12:fpr128 = PMULLv16i8 undef $q0, undef $q1
+    %13:fpr128 = EORv16i8 %12, undef $q2
+
+    ; CHECK: SU(14): %14:fpr128 = PMULLv8i8 undef $d0, undef $d1
+    ; CHECK: Successors:
+    ; NOFUSE-NOT: SU({{.*}}): Ord
+    ; FUSEAES-NOT: SU({{.*}}): Ord
+    ; FUSECRYPTO: SU(15): Ord  Latency=0 Cluster
+    ; CHECK: SU(15): dead %15:fpr128 = EORv16i8 %14:fpr128, undef $q3
+    %14:fpr128 = PMULLv8i8 undef $d0, undef $d1
+    %15:fpr128 = EORv16i8 %14, undef $q3