[Sparc][LEON] Hardware erratum fix: Insert NOPs around double precision instructions

[lero_chris]

Summary

Fixes an erratum in LEON 2 processors where the result of double precision operations is not immediately available. This is fixed by inserting NOP instructions around the erroneous instructions.

The error is described by the LEON erratum: "Odd-Numbered FPU Register Dependency not Properly Checked in some Double-Precision FPU Operations”

Unit test included.

[dcederman]

lib/Target/Sparc/LeonPasses.cpp

281

It should be a single precision load according to the errata document.

From http://www.atmel.com/Images/doc4409.pdf :
"Data dependency is not properly checked between a load singleword floating - point instruction (LDF) involving an odd-numbered floating-point register as a destination of the load and an immediately following double-precision floating-point instruction".

338–339

The indentation looks wrong here. You can use clang format to automatically indent and format your patch.

lib/Target/Sparc/LeonFeatures.td

 
 def InsertNOPLoad: SubtargetFeature<
   "insertnopload",
   "InsertNOPLoad",
   "true",
   "LEON3 erratum fix: Insert a NOP instruction after every single-cycle load instruction when the next instruction is another load/store instruction" 
 >;
 
+def InsertNOPDoublePrecision : SubtargetFeature<
+  "insrtnopdblprcsn",
+  "InsertNOPDoublePrecision",
+  "true",
+  "LEON2 erratum fix: Insert a NOP before the double "
+  "precision floating point instruction"
+>;
+                       
 def FixFSMULD : SubtargetFeature<
   "fixfsmuld",
   "FixFSMULD",
   "true",
   "LEON erratum fix: Do not use FSMULD" 
 >;
 
 def ReplaceFMULS : SubtargetFeature<

lib/Target/Sparc/LeonPasses.h

public:
 
   const char *getPassName() const override {
     return "InsertNOPLoad: Erratum Fix LBR35: insert a NOP instruction after "
            "every single-cycle load instruction when the next instruction is "
            "another load/store instruction";
   }
 };
 
+class LLVM_LIBRARY_VISIBILITY InsertNOPDoublePrecision
+    : public LEONMachineFunctionPass {
+public:
+  static char ID;
+
+  InsertNOPDoublePrecision(TargetMachine &tm);
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  const char *getPassName() const override {
+    return "InsertNOPDoublePrecision: Leon erratum fix: insert a NOP before "
+           "the double precision floating point instruction";
+  }
+};
+
 class LLVM_LIBRARY_VISIBILITY FixFSMULD : public LEONMachineFunctionPass {
 public:
   static char ID;
 
   FixFSMULD(TargetMachine &tm);
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   const char *getPassName() const override {

lib/Target/Sparc/LeonPasses.cpp

for (auto MBBI = MBB.begin(), E = MBB.end(); MBBI != E; ++MBBI) {
       }
     }
   }
 
   return Modified;
 }
 
 //*****************************************************************************
+//**** InsertNOPDoublePrecision pass
+//*****************************************************************************
+// This erratum fix for some earlier LEON processors fixes a problem where a
+// double precision load will not yield the correct result if used in FMUL,
+// FDIV, FADD, FSUB or FSQRT instructions later. If this sequence is detected,
+// inserting a NOP between the two instructions will fix the erratum.
+// 1.scans the code after register allocation;
+// 2.checks for the problem conditions as described in the AT697E erratum
+// “Odd-Numbered FPU Register Dependency not Properly Checked in some
+// Double-Precision FPU Operations”;
+// 3.inserts NOPs if the problem exists.
+//
+char InsertNOPDoublePrecision::ID = 0;
+
+InsertNOPDoublePrecision::InsertNOPDoublePrecision(TargetMachine &tm)
+    : LEONMachineFunctionPass(tm, ID) {}
+
+bool InsertNOPDoublePrecision::runOnMachineFunction(MachineFunction &MF) {
+  Subtarget = &MF.getSubtarget<SparcSubtarget>();
+  const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
+  DebugLoc DL = DebugLoc();
+
+  bool Modified = false;
+  for (auto MFI = MF.begin(), E = MF.end(); MFI != E; ++MFI) {
+    MachineBasicBlock &MBB = *MFI;
+    for (auto MBBI = MBB.begin(), E = MBB.end(); MBBI != E; ++MBBI) {
+      MachineInstr &MI = *MBBI;
+      unsigned Opcode = MI.getOpcode();
+      if (Opcode == SP::LDDFri || Opcode == SP::LDDFrr) {
+        MachineBasicBlock::iterator NMBBI = std::next(MBBI);
+        MachineInstr &NMI = *NMBBI;
+
+        unsigned NextOpcode = NMI.getOpcode();
+        if (NextOpcode == SP::FADDD || NextOpcode == SP::FSUBD ||
+            NextOpcode == SP::FMULD || NextOpcode == SP::FDIVD) {
+          int RegAIndex = GetRegIndexForOperand(MI, 0);
+          int RegBIndex = GetRegIndexForOperand(NMI, 0);
+          int RegCIndex =
+              GetRegIndexForOperand(NMI, 2); // Second source operand is index 2
+          int RegDIndex =
+              GetRegIndexForOperand(NMI, 1); // Destination operand is index 1
+
+          if ((RegAIndex == RegBIndex + 1 && RegBIndex == RegDIndex) ||
+        (RegAIndex == RegCIndex + 1 && RegCIndex == RegDIndex) ||
+        (RegAIndex == RegBIndex + 1 && RegCIndex == RegDIndex) ||
+        (RegAIndex == RegCIndex + 1 && RegBIndex == RegDIndex)) {
+      // Insert NOP between the two instructions.
+      BuildMI(MBB, NMBBI, DL, TII.get(SP::NOP));
+      Modified = true;
+      }
+
+      // Check the errata patterns that only happen for FADDD and FMULD
+      if (Modified == false &&
+        (NextOpcode == SP::FADDD || NextOpcode == SP::FMULD)) {
+      RegAIndex = GetRegIndexForOperand(MI, 1);
+      if (RegAIndex == RegBIndex + 1 && RegBIndex == RegCIndex &&
+        RegBIndex == RegDIndex) {
+        // Insert NOP between the two instructions.
+        BuildMI(MBB, NMBBI, DL, TII.get(SP::NOP));
+        Modified = true;
+      }
+      }
+    } else if (NextOpcode == SP::FSQRTD) {
+      int RegAIndex = GetRegIndexForOperand(MI, 1);
+      int RegBIndex = GetRegIndexForOperand(NMI, 0);
+      int RegCIndex = GetRegIndexForOperand(NMI, 1);
+
+      if (RegAIndex == RegBIndex + 1 && RegBIndex == RegCIndex) {
+      // Insert NOP between the two instructions.
+      BuildMI(MBB, NMBBI, DL, TII.get(SP::NOP));
+      Modified = true;
+      }
+    }
+    }
+  }
+  }
+
+  return Modified;
+}
+
+
+//*****************************************************************************
 //**** FixAllFDIVSQRT pass
 //*****************************************************************************
 // This pass fixes the incorrectly working FDIVx and FSQRTx instructions that
 // exist for some earlier versions of the LEON processor line. Five NOP
 // instructions need to be inserted after these instructions to ensure the
 // correct result is placed in the destination registers before they are used.
 //
 // This pass implements two fixes:

lib/Target/Sparc/Sparc.td

 
 // LEON 2 FT generic
 def : Processor<"leon2", LEON2Itineraries,
                 [FeatureLeon]>;
 
 // LEON 2 FT (AT697E)
 // TO DO: Place-holder: Processor specific features will be added *very* soon here.
 def : Processor<"at697e", LEON2Itineraries,
-                [FeatureLeon, InsertNOPLoad]>;
+                [FeatureLeon, InsertNOPLoad, InsertNOPDoublePrecision]>;
 
 // LEON 2 FT (AT697F)
 // TO DO: Place-holder: Processor specific features will be added *very* soon here.
 def : Processor<"at697f", LEON2Itineraries,
-                [FeatureLeon, InsertNOPLoad]>;
+                [FeatureLeon, InsertNOPLoad, InsertNOPDoublePrecision]>;
 
 
 // LEON 3 FT generic
 def : Processor<"leon3", LEON3Itineraries,
                 [FeatureLeon, UMACSMACSupport]>;
 
 // LEON 3 FT (UT699). Provides features for the UT699 processor
 // - covers all the erratum fixes for LEON3, but does not support the CASA instruction.

lib/Target/Sparc/SparcSubtarget.h

class SparcSubtarget : public SparcGenSubtargetInfo {
 
   // LEON features
   bool HasUmacSmac;
   bool HasLeonCasa;
   bool InsertNOPLoad;
   bool FixFSMULD;
   bool ReplaceFMULS;
   bool FixAllFDIVSQRT;
+  bool InsertNOPDoublePrecision;
 
   SparcInstrInfo InstrInfo;
   SparcTargetLowering TLInfo;
   SelectionDAGTargetInfo TSInfo;
   SparcFrameLowering FrameLowering;
 
 public:
   SparcSubtarget(const Triple &TT, const std::string &CPU,
public:
   bool hasHardQuad() const { return HasHardQuad; }
   bool usePopc() const { return UsePopc; }
   bool useSoftFloat() const { return UseSoftFloat; }
 
   // Leon options
   bool hasUmacSmac() const { return HasUmacSmac; }
   bool hasLeonCasa() const { return HasLeonCasa; }
   bool insertNOPLoad() const { return InsertNOPLoad; }
+  bool insertNOPDoublePrecision() const { return InsertNOPDoublePrecision; }
   bool fixFSMULD() const { return FixFSMULD; }
   bool replaceFMULS() const { return ReplaceFMULS; }
   bool fixAllFDIVSQRT() const { return FixAllFDIVSQRT; }
 
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
   SparcSubtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS);

lib/Target/Sparc/SparcSubtarget.cpp

SparcSubtarget &SparcSubtarget::initializeSubtargetDependencies(StringRef CPU,
   IsVIS = false;
   HasHardQuad = false;
   UsePopc = false;
   UseSoftFloat = false;
 
   // Leon features
   HasLeonCasa = false;
   HasUmacSmac = false;
+  InsertNOPDoublePrecision = false;
   InsertNOPLoad = false;
   FixFSMULD = false;
   ReplaceFMULS = false;
   FixAllFDIVSQRT = false;
 
   // Determine default and user specified characteristics
   std::string CPUName = CPU;
   if (CPUName.empty())

lib/Target/Sparc/SparcTargetMachine.cpp

void SparcPassConfig::addPreEmitPass(){
   if (this->getSparcTargetMachine().getSubtargetImpl()->replaceFMULS())
   {
     addPass(new ReplaceFMULS(getSparcTargetMachine()));
   }
   if (this->getSparcTargetMachine().getSubtargetImpl()->fixAllFDIVSQRT())
   {
     addPass(new FixAllFDIVSQRT(getSparcTargetMachine()));
   }
+  if (this->getSparcTargetMachine()
+      .getSubtargetImpl()->insertNOPDoublePrecision()) {
+    addPass(new InsertNOPDoublePrecision(getSparcTargetMachine()));
+  }
 }
 
 void SparcV8TargetMachine::anchor() { }
 
 SparcV8TargetMachine::SparcV8TargetMachine(const Target &T, const Triple &TT,
                                            StringRef CPU, StringRef FS,
                                            const TargetOptions &Options,
                                            Optional<Reloc::Model> RM,

test/CodeGen/SPARC/LeonInsertNOPsDoublePrecision.ll

+; RUN: llc %s -O0 -march=sparc -mcpu=at697f -o - | FileCheck %s
+
+; CHECK:        ldd
+; CHECK:        ldd
+; CHECK-NEXT:   nop
+
+define double @mult() #0 {
+entry:
+  %x = alloca double, align 8
+  %y = alloca double, align 8
+  store double 3.141590e+00, double* %x, align 8
+  store double 1.234560e+00, double* %y, align 8
+  %0 = load double, double* %x, align 8
+  %1 = load double, double* %y, align 8
+  %mul = fmul double %0, %1
+  ret double %mul
+}