[AMDGPU] need to insert wait between the scalar load and vector store to the same address to avoid WAR conflict.

[alex-t]

Summary

Before divergence driven ISel introduced scalar loads from the global address space we relied on the VMEM operations ordering enforced by the HW. Now we can easily get WAR on scalar load followed vector store to same address.
The case is here: https://github.com/RadeonOpenCompute/ROCm/issues/500

Current fix relies on the MachineMemOperands equality to check that SMRD and VMEM use same address.
Proper fix should include creating the alias analysis on the machine IR that is obviously too big hummer at the moment.

[rampitec]

llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll

1

Please add -check-prefix=GCN.

8

Test should have no numbered values. Please run opt -instnamer on it.

[alex-t]

Test updated.

[rampitec]

LGTM. Delete source_filename from the test before push.

llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll

9

Delete this line.

llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp

 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>

 private:
   const GCNSubtarget *ST = nullptr;
   const SIInstrInfo *TII = nullptr;
   const SIRegisterInfo *TRI = nullptr;
   const MachineRegisterInfo *MRI = nullptr;
   AMDGPU::IsaVersion IV;
 
   DenseSet<MachineInstr *> TrackedWaitcntSet;
+  DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
+  MachinePostDominatorTree *PDT;
 
   struct BlockInfo {
     MachineBasicBlock *MBB;
     std::unique_ptr<WaitcntBrackets> Incoming;
     bool Dirty = true;
 
     explicit BlockInfo(MachineBasicBlock *MBB) : MBB(MBB) {}
   };
public:
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   StringRef getPassName() const override {
     return "SI insert wait instructions";
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
+    AU.addRequired<MachinePostDominatorTree>();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
   bool isForceEmitWaitcnt() const {
     for (auto T : inst_counter_types())
       if (ForceEmitWaitcnt[T])
         return true;
     return false;
bool WaitcntBrackets::counterOutOfOrder(InstCounterType T) const {
   // Scalar memory read always can go out of order.
   if (T == LGKM_CNT && hasPendingEvent(SMEM_ACCESS))
     return true;
   return MixedPendingEvents[T];
 }
 
 INITIALIZE_PASS_BEGIN(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false,
                       false)
+INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
 INITIALIZE_PASS_END(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false,
                     false)
 
 char SIInsertWaitcnts::ID = 0;
 
 char &llvm::SIInsertWaitcntsID = SIInsertWaitcnts::ID;
 
 FunctionPass *llvm::createSIInsertWaitcntsPass() {
if (MI.isCall() && callWaitsOnFunctionEntry(MI)) {
       // Two cases are handled for destination operands:
       // 1) If the destination operand was defined by a load, add the s_waitcnt
       // instruction to guarantee the right WAW order.
       // 2) If a destination operand that was used by a recent export/store ins,
       // add s_waitcnt on exp_cnt to guarantee the WAR order.
       if (MI.mayStore()) {
         // FIXME: Should not be relying on memoperands.
         for (const MachineMemOperand *Memop : MI.memoperands()) {
+          const Value *Ptr = Memop->getValue();
+          if (SLoadAddresses.count(Ptr)) {
+            addWait(Wait, LGKM_CNT, 0);
+            if (PDT->dominates(MI.getParent(),
+                               SLoadAddresses.find(Ptr)->second))
+              SLoadAddresses.erase(Ptr);
+          }
           unsigned AS = Memop->getAddrSpace();
           if (AS != AMDGPUAS::LOCAL_ADDRESS)
             continue;
           unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
           ScoreBrackets.determineWait(
               VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
           ScoreBrackets.determineWait(
               EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait);
if (readsVCCZ(Inst)) {
       if (ScoreBrackets.getScoreLB(LGKM_CNT) <
               ScoreBrackets.getScoreUB(LGKM_CNT) &&
           ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
         if (ST->hasReadVCCZBug())
           VCCZBugWorkAround = true;
       }
     }
 
+    if (TII->isSMRD(Inst)) {
+      for (const MachineMemOperand *Memop : Inst.memoperands()) {
+        const Value *Ptr = Memop->getValue();
+        SLoadAddresses.insert(std::make_pair(Ptr, Inst.getParent()));
+      }
+    }
+
     // Generate an s_waitcnt instruction to be placed before
     // cur_Inst, if needed.
     Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr);
     OldWaitcntInstr = nullptr;
 
     updateEventWaitcntAfter(Inst, &ScoreBrackets);
 
 #if 0 // TODO: implement resource type check controlled by options with ub = LB.

 
 bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
   ST = &MF.getSubtarget<GCNSubtarget>();
   TII = ST->getInstrInfo();
   TRI = &TII->getRegisterInfo();
   MRI = &MF.getRegInfo();
   IV = AMDGPU::getIsaVersion(ST->getCPU());
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  PDT = &getAnalysis<MachinePostDominatorTree>();
 
   ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
   for (auto T : inst_counter_types())
     ForceEmitWaitcnt[T] = false;
 
   HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV);
   HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
   HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);

llvm/test/CodeGen/AMDGPU/smrd_vmem_war.ll

+; RUN: llc  -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s
+
+; CHECK-LABEL: BB0_1
+; CHECK: s_load_dword s{{[0-9]+}}, s{{\[}}[[ADDR_LO:[0-9]+]]{{\:}}[[ADDR_HI:[0-9]+]]{{\]}}, 0x0
+; CHECK: s_waitcnt lgkmcnt(0)
+; CHECK: global_store_dword v{{\[}}[[ADDR_LO]]{{\:}}[[ADDR_HI]]{{\]}}, v{{[0-9]+}}, off
+
+define amdgpu_kernel void @smrd_vmem_war(i32 addrspace(1)* nocapture %0, i64 addrspace(1)* nocapture %1) {
+  %3 = call i32 @llvm.amdgcn.workitem.id.x()
+  %4 = icmp eq i32 %3, 0
+  br i1 %4, label %5, label %10
+
+5:                                                ; preds = %2
+  %6 = load i32, i32 addrspace(1)* %0, align 4
+  store i32 0, i32 addrspace(1)* %0, align 4
+  %7 = zext i32 %6 to i64
+  %8 = load i64, i64 addrspace(1)* %1, align 8
+  %9 = add i64 %8, %7
+  store i64 %9, i64 addrspace(1)* %1, align 8
+  br label %10
+
+10:                                               ; preds = %5, %2
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()