[AArch64] Add new target feature to fuse literal generation

[evandro]

Summary

This feature enables the fusion of such operations on Cortex A57, as recommended in its Software Optimisation Guide, sections 4.14 and 4.15.

[rengolin]

Small comment on the test, otherwise, looks good.

llvm/test/CodeGen/AArch64/misched-fusion-lit.ll

2

Can you add a line that disables fusing (either A53 or mattr)? Just to make sure we check for the right things on both sides.

[rengolin]

Thanks! LGTM. I think D28697 should be fine, too, but I'd rather Kristof/James/Tim would have an extra ack.

cheers,
--renato

[evandro]

Final patch after approval.

llvm/lib/Target/AArch64/AArch64.td

 def FeatureArithmeticCbzFusion : SubtargetFeature<
     "arith-cbz-fusion", "HasArithmeticCbzFusion", "true",
     "CPU fuses arithmetic + cbz/cbnz operations">;
 
 def FeatureFuseAES : SubtargetFeature<
     "fuse-aes", "HasFuseAES", "true",
     "CPU fuses AES crypto operations">;
 
+def FeatureFuseLiterals : SubtargetFeature<
+    "fuse-literals", "HasFuseLiterals", "true",
+    "CPU fuses literal generation operations">;
+
 def FeatureDisableLatencySchedHeuristic : SubtargetFeature<
     "disable-latency-sched-heuristic", "DisableLatencySchedHeuristic", "true",
     "Disable latency scheduling heuristic">;
 
 def FeatureUseRSqrt : SubtargetFeature<
     "use-reciprocal-square-root", "UseRSqrt", "true",
     "Use the reciprocal square root approximation">;
 
def ProcA53     : SubtargetFeature<"a53", "ARMProcFamily", "CortexA53",
                                    FeaturePerfMon,
                                    FeaturePostRAScheduler,
                                    FeatureUseAA
                                    ]>;
 
 def ProcA57     : SubtargetFeature<"a57", "ARMProcFamily", "CortexA57",
                                    "Cortex-A57 ARM processors", [
                                    FeatureFuseAES,
+                                   FeatureFuseLiterals,
                                    FeatureBalanceFPOps,
                                    FeatureCRC,
                                    FeatureCrypto,
                                    FeatureCustomCheapAsMoveHandling,
                                    FeatureFPARMv8,
                                    FeatureNEON,
                                    FeaturePerfMon,
                                    FeaturePostRAScheduler,

llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

bool AArch64InstrInfo::shouldScheduleAdjacent(
   if (Subtarget.hasFuseAES())
     // Fuse AES crypto operations:
     // - AES encode
     // - AES decode
     if ((FirstOpcode == AArch64::AESErr && SecondOpcode == AArch64::AESMCrr) ||
         (FirstOpcode == AArch64::AESDrr && SecondOpcode == AArch64::AESIMCrr))
       return true;
 
+  if (Subtarget.hasFuseLiterals())
+    // Fuse literal generation operations:
+    // - PC relative addresses
+    // - large immediates
+    if ((FirstOpcode == AArch64::ADRP && SecondOpcode == AArch64::ADDXri) ||
+        ((FirstOpcode == AArch64::MOVZWi && SecondOpcode == AArch64::MOVKWi &&
+          Second.getOperand(3).getImm() == 16) ||
+         (FirstOpcode == AArch64::MOVZXi && SecondOpcode == AArch64::MOVKXi &&
+          Second.getOperand(3).getImm() == 16) ||
+         (FirstOpcode == AArch64::MOVKXi && SecondOpcode == AArch64::MOVKXi &&
+          First.getOperand(3).getImm() == 32 &&
+          Second.getOperand(3).getImm() == 48)))
+      return true;
+
   return false;
 }
 
 MachineInstr *AArch64InstrInfo::emitFrameIndexDebugValue(
     MachineFunction &MF, int FrameIx, uint64_t Offset, const MDNode *Var,
     const MDNode *Expr, const DebugLoc &DL) const {
   MachineInstrBuilder MIB = BuildMI(MF, DL, get(AArch64::DBG_VALUE))
                                 .addFrameIndex(FrameIx)

llvm/lib/Target/AArch64/AArch64Subtarget.h

protected:
   bool CustomAsCheapAsMove = false;
   bool UsePostRAScheduler = false;
   bool Misaligned128StoreIsSlow = false;
   bool AvoidQuadLdStPairs = false;
   bool UseAlternateSExtLoadCVTF32Pattern = false;
   bool HasArithmeticBccFusion = false;
   bool HasArithmeticCbzFusion = false;
   bool HasFuseAES = false;
+  bool HasFuseLiterals = false;
   bool DisableLatencySchedHeuristic = false;
   bool UseRSqrt = false;
   uint8_t MaxInterleaveFactor = 2;
   uint8_t VectorInsertExtractBaseCost = 3;
   uint16_t CacheLineSize = 0;
   uint16_t PrefetchDistance = 0;
   uint16_t MinPrefetchStride = 1;
   unsigned MaxPrefetchIterationsAhead = UINT_MAX;
public:
   bool isMisaligned128StoreSlow() const { return Misaligned128StoreIsSlow; }
   bool avoidQuadLdStPairs() const { return AvoidQuadLdStPairs; }
   bool useAlternateSExtLoadCVTF32Pattern() const {
     return UseAlternateSExtLoadCVTF32Pattern;
   }
   bool hasArithmeticBccFusion() const { return HasArithmeticBccFusion; }
   bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; }
   bool hasFuseAES() const { return HasFuseAES; }
+  bool hasFuseLiterals() const { return HasFuseLiterals; }
   bool useRSqrt() const { return UseRSqrt; }
   unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; }
   unsigned getVectorInsertExtractBaseCost() const {
     return VectorInsertExtractBaseCost;
   }
   unsigned getCacheLineSize() const { return CacheLineSize; }
   unsigned getPrefetchDistance() const { return PrefetchDistance; }
   unsigned getMinPrefetchStride() const { return MinPrefetchStride; }

llvm/lib/Target/AArch64/AArch64TargetMachine.cpp

public:
   createMachineScheduler(MachineSchedContext *C) const override {
     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
     DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
     DAG->addMutation(createMacroFusionDAGMutation(DAG->TII));
     return DAG;
   }
 
+  ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const override {
+    const AArch64Subtarget &ST = C->MF->getSubtarget<AArch64Subtarget>();
+    if (ST.hasFuseLiterals()) {
+      // Run the Macro Fusion after RA again since literals are expanded from
+      // pseudos then (v. addPreSched2()).
+      ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
+      DAG->addMutation(createMacroFusionDAGMutation(DAG->TII));
+      return DAG;
+    }
+
+    return nullptr;
+  }
+
   void addIRPasses()  override;
   bool addPreISel() override;
   bool addInstSelector() override;
 #ifdef LLVM_BUILD_GLOBAL_ISEL
   bool addIRTranslator() override;
   bool addLegalizeMachineIR() override;
   bool addRegBankSelect() override;
   bool addGlobalInstructionSelect() override;

llvm/test/CodeGen/AArch64/misched-fusion-lit.ll

+; RUN: llc %s -o - -mtriple=aarch64-unknown -mattr=fuse-literals | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-a57     | FileCheck %s
+
+@g = common local_unnamed_addr global i8* null, align 8
+
+define i8* @litp(i32 %a, i32 %b) {
+entry:
+  %add = add nsw i32 %b, %a
+  %idx.ext = sext i32 %add to i64
+  %add.ptr = getelementptr i8, i8* bitcast (i8* (i32, i32)* @litp to i8*), i64 %idx.ext
+  store i8* %add.ptr, i8** @g, align 8
+  ret i8* %add.ptr
+
+; CHECK-LABEL: litp:
+; CHECK: adrp [[R:x[0-9]+]], litp
+; CHECK-NEXT: add {{x[0-9]+}}, [[R]], :lo12:litp
+}
+
+define i32 @liti(i32 %a, i32 %b) {
+entry:
+  %add = add i32 %a, -262095121
+  %add1 = add i32 %add, %b
+  ret i32 %add1
+
+; CHECK-LABEL: liti:
+; CHECK: mov [[R:w[0-9]+]], {{#[0-9]+}}
+; CHECK-NEXT: movk [[R]], {{#[0-9]+}}, lsl #16
+}
+
+; Function Attrs: norecurse nounwind readnone
+define i64 @litl(i64 %a, i64 %b) {
+entry:
+  %add = add i64 %a, 2208998440489107183
+  %add1 = add i64 %add, %b
+  ret i64 %add1
+
+; CHECK-LABEL: litl:
+; CHECK: mov [[RA:x[0-9]+]], {{#[0-9]+}}
+; CHECK-NEXT: movk [[RA]], {{#[0-9]+}}, lsl #16
+; CHECK: movk [[RB:x[0-9]+]], {{#[0-9]+}}, lsl #32
+; CHECK-NEXT: movk [[RB]], {{#[0-9]+}}, lsl #48
+}
+
+