[ARM] Add new target feature to fuse literal generation

[evandro]

Summary

This feature enables the fusion of such operations on Cortex A57 and Cortex A72, as recommended in their Software Optimisation Guides, sections 4.14 and 4.11, respectively.

[rengolin]

Hi Evandro, looks great, short and simple!

Shouldn't you also add the option to the CPUs description?

[evandro]

In D49563#1168686, @rengolin wrote:

Shouldn't you also add the option to the CPUs description?

I shall, at least for Exynos, but I'd like to hear from our friends at ARM too.

[rengolin]

In D49563#1168689, @evandro wrote:

I shall, at least for Exynos, but I'd like to hear from our friends at ARM too.

Makes sense. Adding a few more people, then.

[SjoerdMeijer]

@fhahn, if I remember correctly, you've worked on similar things not that long ago. Can you have a look?

[evandro]

Ping! 🔔

[fhahn]

LGTM, this is in line with the Cortex-A57 and Cortex-A72 opt guides. We have a similar feature for AArch64 already.

llvm/lib/Target/ARM/ARM.td

145

FeatureFuseLiterals should be added to the cortex-a57 and cortex-a72 features according to the optimization guides. Can you also add a run lines with those CPUs to the test?

[evandro]

llvm/lib/Target/ARM/ARM.td

145

It's more complicated than that. I had to enable FeatureUseMISched as well for this to be effective. This is fine for Exynos, but it'd require more extensive testing on Cortex.

[fhahn]

llvm/lib/Target/ARM/ARM.td

145

Ah right! No worries then

[evandro]

llvm/lib/Target/ARM/ARM.td

145

Of course, Exynos needs this feature because there's no pipeline model for it in the ARM target.

llvm/lib/Target/ARM/ARM.td

 def FeatureFPAO           : SubtargetFeature<"fpao", "HasFPAO", "true",
                                              "Enable fast computation of "
                                              "positive address offsets">;
 
 // Fast execution of AES crypto operations
 def FeatureFuseAES        : SubtargetFeature<"fuse-aes", "HasFuseAES", "true",
                                              "CPU fuses AES crypto operations">;
 
+// Fast execution of bottom and top halves of literal generation
+def FeatureFuseLiterals   : SubtargetFeature<"fuse-literals", "HasFuseLiterals", "true",
+                                             "CPU fuses literal generation operations">;
+
 // The way of reading thread pointer                                             
 def FeatureReadTp :  SubtargetFeature<"read-tp-hard", "ReadTPHard", "true",
                                       "Reading thread pointer from register">;
 
 // Cyclone can zero VFP registers in 0 cycles.
 def FeatureZCZeroing      : SubtargetFeature<"zcz", "HasZeroCycleZeroing", "true",
                                              "Has zero-cycle zeroing instructions">;
 

llvm/lib/Target/ARM/ARMMacroFusion.cpp

 
 #include "ARMMacroFusion.h"
 #include "ARMSubtarget.h"
 #include "llvm/CodeGen/MacroFusion.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 
 namespace llvm {
 
-/// Check if the instr pair, FirstMI and SecondMI, should be fused
-/// together. Given SecondMI, when FirstMI is unspecified, then check if
-/// SecondMI may be part of a fused pair at all.
-static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
-                                   const TargetSubtargetInfo &TSI,
-                                   const MachineInstr *FirstMI,
-                                   const MachineInstr &SecondMI) {
-  const ARMSubtarget &ST = static_cast<const ARMSubtarget&>(TSI);
-
-  // Assume wildcards for unspecified instrs.
+// Fuse AES crypto encoding or decoding.
+static bool isAESPair(const MachineInstr *FirstMI,
+  // Assume the 1st instr to be a wildcard if it is unspecified.
   unsigned FirstOpcode =
       FirstMI ? FirstMI->getOpcode()
               : static_cast<unsigned>(ARM::INSTRUCTION_LIST_END);
   unsigned SecondOpcode = SecondMI.getOpcode();
 
-  if (ST.hasFuseAES())
-    // Fuse AES crypto operations.
-    switch(SecondOpcode) {
-    // AES encode.
-    case ARM::AESMC :
-      return FirstOpcode == ARM::AESE ||
-             FirstOpcode == ARM::INSTRUCTION_LIST_END;
-    // AES decode.
-    case ARM::AESIMC:
-      return FirstOpcode == ARM::AESD ||
-             FirstOpcode == ARM::INSTRUCTION_LIST_END;
-    }
 
   return false;
 }
 
+// Fuse literal generation.
+static bool isLiteralsPair(const MachineInstr *FirstMI,
+                           const MachineInstr &SecondMI) {
+  // Assume the 1st instr to be a wildcard if it is unspecified.
+  unsigned FirstOpcode =
+      FirstMI ? FirstMI->getOpcode()
+              : static_cast<unsigned>(ARM::INSTRUCTION_LIST_END);
+  unsigned SecondOpcode = SecondMI.getOpcode();
+
+  // 32 bit immediate.
+  if ((FirstOpcode == ARM::INSTRUCTION_LIST_END ||
+       FirstOpcode == ARM::MOVi16) &&
+      SecondOpcode == ARM::MOVTi16)
+    return true;
+
+  return false;
+}
+
+/// Check if the instr pair, FirstMI and SecondMI, should be fused
+/// together. Given SecondMI, when FirstMI is unspecified, then check if
+/// SecondMI may be part of a fused pair at all.
+static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
+                                   const TargetSubtargetInfo &TSI,
+                                   const MachineInstr *FirstMI,
+                                   const MachineInstr &SecondMI) {
+  const ARMSubtarget &ST = static_cast<const ARMSubtarget&>(TSI);
+
+  if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
+    return true;
+  if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
+    return true;
+
+  return false;
+}
+
 std::unique_ptr<ScheduleDAGMutation> createARMMacroFusionDAGMutation () {
   return createMacroFusionDAGMutation(shouldScheduleAdjacent);
 }
 
 } // end namespace llvm

llvm/lib/Target/ARM/ARMSubtarget.h

protected:
 
   /// HasFPAO - if true, processor  does positive address offset computation faster
   bool HasFPAO = false;
 
   /// HasFuseAES - if true, processor executes back to back AES instruction
   /// pairs faster.
   bool HasFuseAES = false;
 
+  /// HasFuseLiterals - if true, processor executes back to back
+  /// bottom and top halves of literal generation faster.
+  bool HasFuseLiterals = false;
+
   /// If true, if conversion may decide to leave some instructions unpredicated.
   bool IsProfitableToUnpredicate = false;
 
   /// If true, VMOV will be favored over VGETLNi32.
   bool HasSlowVGETLNi32 = false;
 
   /// If true, VMOV will be favored over VDUP.
   bool HasSlowVDUP32 = false;
public:
   bool genLongCalls() const { return GenLongCalls; }
   bool genExecuteOnly() const { return GenExecuteOnly; }
 
   bool hasFP16() const { return HasFP16; }
   bool hasD16() const { return HasD16; }
   bool hasFullFP16() const { return HasFullFP16; }
 
   bool hasFuseAES() const { return HasFuseAES; }
+  bool hasFuseLiterals() const { return HasFuseLiterals; }
   /// Return true if the CPU supports any kind of instruction fusion.
-  bool hasFusion() const { return hasFuseAES(); }
+  bool hasFusion() const { return hasFuseAES() || hasFuseLiterals(); }
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
   bool isTargetIOS() const { return TargetTriple.isiOS(); }
   bool isTargetWatchOS() const { return TargetTriple.isWatchOS(); }
   bool isTargetWatchABI() const { return TargetTriple.isWatchABI(); }
   bool isTargetLinux() const { return TargetTriple.isOSLinux(); }

llvm/test/CodeGen/ARM/misched-fusion-lit.ll

+; RUN: llc %s -o - -mtriple=armv8-unknown -mattr=-fuse-literals,+use-misched | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKDONT
+; RUN: llc %s -o - -mtriple=armv8-unknown -mattr=+fuse-literals,+use-misched | FileCheck %s --check-prefix=CHECK --check-prefix=CHECKFUSE
+
+@g = common global i32* zeroinitializer
+
+define i32* @litp(i32 %a, i32 %b) {
+entry:
+  %add = add nsw i32 %b, %a
+  %ptr = getelementptr i32, i32* bitcast (i32* (i32, i32)* @litp to i32*), i32 %add
+  %res = getelementptr i32, i32* bitcast (i32** @g to i32*), i32 %add
+  store i32* %ptr, i32** @g, align 4
+  ret i32* %res
+
+; CHECK-LABEL: litp:
+; CHECK:          movw [[R:r[0-9]+]], :lower16:litp
+; CHECKDONT-NEXT: movw [[S:r[0-9]+]], :lower16:g
+; CHECKFUSE-NEXT: movt [[R]], :upper16:litp
+; CHECKFUSE-NEXT: movw [[S:r[0-9]+]], :lower16:g
+; CHECKFUSE-NEXT: movt [[S]], :upper16:g
+}
+
+define i32 @liti(i32 %a, i32 %b) {
+entry:
+  %adda = add i32 %a, -262095121
+  %add1 = add i32 %adda, %b
+  %addb = add i32 %b, 121110837
+  %add2 = add i32 %addb, %a
+  store i32 %add1, i32* bitcast (i32** @g to i32*), align 4
+  ret i32 %add2
+
+; CHECK-LABEL: liti:
+; CHECK:          movw [[R:r[0-9]+]], #309
+; CHECKDONT-NEXT: add {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}
+; CHECKFUSE-NEXT: movt [[R]], #1848
+; CHECKFUSE:      movw [[S:r[0-9]+]], :lower16:g
+; CHECKFUSE-NEXT: movt [[S]], :upper16:g
+; CHECKFUSE-NEXT: movw [[T:r[0-9]+]], #48879
+; CHECKFUSE-NEXT: movt [[T]], #61536
+}