This is an archive of the discontinued LLVM Phabricator instance.

	Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines
	FunctionPass *createSIShrinkInstructionsPass();			FunctionPass *createSIShrinkInstructionsPass();
	FunctionPass *createSILoadStoreOptimizerPass();			FunctionPass *createSILoadStoreOptimizerPass();
	FunctionPass *createSIWholeQuadModePass();			FunctionPass *createSIWholeQuadModePass();
	FunctionPass *createSIFixControlFlowLiveIntervalsPass();			FunctionPass *createSIFixControlFlowLiveIntervalsPass();
	FunctionPass *createSIOptimizeExecMaskingPreRAPass();			FunctionPass *createSIOptimizeExecMaskingPreRAPass();
	FunctionPass *createSIFixSGPRCopiesPass();			FunctionPass *createSIFixSGPRCopiesPass();
	FunctionPass *createSIMemoryLegalizerPass();			FunctionPass *createSIMemoryLegalizerPass();
	FunctionPass *createSIDebuggerInsertNopsPass();			FunctionPass *createSIDebuggerInsertNopsPass();
	FunctionPass *createSIInsertWaitsPass();
	FunctionPass *createSIInsertWaitcntsPass();			FunctionPass *createSIInsertWaitcntsPass();
	FunctionPass *createSIFixWWMLivenessPass();			FunctionPass *createSIFixWWMLivenessPass();
	FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetOptions &);			FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetOptions &);
	FunctionPass *createAMDGPUUseNativeCallsPass();			FunctionPass *createAMDGPUUseNativeCallsPass();
	FunctionPass *createAMDGPUCodeGenPreparePass();			FunctionPass *createAMDGPUCodeGenPreparePass();
	FunctionPass *createAMDGPUMachineCFGStructurizerPass();			FunctionPass *createAMDGPUMachineCFGStructurizerPass();
	FunctionPass *createAMDGPURewriteOutArgumentsPass();			FunctionPass *createAMDGPURewriteOutArgumentsPass();

	▲ Show 20 Lines • Show All 102 Lines • ▼ Show 20 Lines
	extern char &SIAnnotateControlFlowPassID;			extern char &SIAnnotateControlFlowPassID;

	void initializeSIMemoryLegalizerPass(PassRegistry&);			void initializeSIMemoryLegalizerPass(PassRegistry&);
	extern char &SIMemoryLegalizerID;			extern char &SIMemoryLegalizerID;

	void initializeSIDebuggerInsertNopsPass(PassRegistry&);			void initializeSIDebuggerInsertNopsPass(PassRegistry&);
	extern char &SIDebuggerInsertNopsID;			extern char &SIDebuggerInsertNopsID;

	void initializeSIInsertWaitsPass(PassRegistry&);
	extern char &SIInsertWaitsID;

	void initializeSIInsertWaitcntsPass(PassRegistry&);			void initializeSIInsertWaitcntsPass(PassRegistry&);
	extern char &SIInsertWaitcntsID;			extern char &SIInsertWaitcntsID;

	void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry&);			void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry&);
	extern char &AMDGPUUnifyDivergentExitNodesID;			extern char &AMDGPUUnifyDivergentExitNodesID;

	ImmutablePass *createAMDGPUAAWrapperPass();			ImmutablePass *createAMDGPUAAWrapperPass();
	void initializeAMDGPUAAWrapperPassPass(PassRegistry&);			void initializeAMDGPUAAWrapperPassPass(PassRegistry&);
	▲ Show 20 Lines • Show All 89 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

Show First 20 Lines • Show All 162 Lines • ▼ Show 20 Lines	extern "C" void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUArgumentUsageInfoPass(*PR);		initializeAMDGPUArgumentUsageInfoPass(*PR);
initializeAMDGPULowerIntrinsicsPass(*PR);		initializeAMDGPULowerIntrinsicsPass(*PR);
initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR);		initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR);
initializeAMDGPUPromoteAllocaPass(*PR);		initializeAMDGPUPromoteAllocaPass(*PR);
initializeAMDGPUCodeGenPreparePass(*PR);		initializeAMDGPUCodeGenPreparePass(*PR);
initializeAMDGPURewriteOutArgumentsPass(*PR);		initializeAMDGPURewriteOutArgumentsPass(*PR);
initializeAMDGPUUnifyMetadataPass(*PR);		initializeAMDGPUUnifyMetadataPass(*PR);
initializeSIAnnotateControlFlowPass(*PR);		initializeSIAnnotateControlFlowPass(*PR);
initializeSIInsertWaitsPass(*PR);
initializeSIInsertWaitcntsPass(*PR);		initializeSIInsertWaitcntsPass(*PR);
initializeSIWholeQuadModePass(*PR);		initializeSIWholeQuadModePass(*PR);
initializeSILowerControlFlowPass(*PR);		initializeSILowerControlFlowPass(*PR);
initializeSIInsertSkipsPass(*PR);		initializeSIInsertSkipsPass(*PR);
initializeSIMemoryLegalizerPass(*PR);		initializeSIMemoryLegalizerPass(*PR);
initializeSIDebuggerInsertNopsPass(*PR);		initializeSIDebuggerInsertNopsPass(*PR);
initializeSIOptimizeExecMaskingPass(*PR);		initializeSIOptimizeExecMaskingPass(*PR);
initializeSIFixWWMLivenessPass(*PR);		initializeSIFixWWMLivenessPass(*PR);
▲ Show 20 Lines • Show All 691 Lines • ▼ Show 20 Lines	void GCNPassConfig::addPreEmitPass() {
// bottom up, so when we begin to schedule a region we don't know what		// bottom up, so when we begin to schedule a region we don't know what
// instructions were emitted directly before it.		// instructions were emitted directly before it.
//		//
// Here we add a stand-alone hazard recognizer pass which can handle all		// Here we add a stand-alone hazard recognizer pass which can handle all
// cases.		// cases.
addPass(&PostRAHazardRecognizerID);		addPass(&PostRAHazardRecognizerID);

addPass(createSIMemoryLegalizerPass());		addPass(createSIMemoryLegalizerPass());
if (EnableSIInsertWaitcntsPass)
addPass(createSIInsertWaitcntsPass());		addPass(createSIInsertWaitcntsPass());
else
addPass(createSIInsertWaitsPass());
addPass(createSIShrinkInstructionsPass());		addPass(createSIShrinkInstructionsPass());
addPass(&SIInsertSkipsPassID);		addPass(&SIInsertSkipsPassID);
addPass(createSIDebuggerInsertNopsPass());		addPass(createSIDebuggerInsertNopsPass());
addPass(&BranchRelaxationPassID);		addPass(&BranchRelaxationPassID);
}		}

TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {		TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
return new GCNPassConfig(*this, PM);		return new GCNPassConfig(*this, PM);
}		}

lib/Target/AMDGPU/CMakeLists.txt

Show First 20 Lines • Show All 76 Lines • ▼ Show 20 Lines	add_llvm_target(AMDGPUCodeGen
SIDebuggerInsertNops.cpp		SIDebuggerInsertNops.cpp
SIFixSGPRCopies.cpp		SIFixSGPRCopies.cpp
SIFixVGPRCopies.cpp		SIFixVGPRCopies.cpp
SIFixWWMLiveness.cpp		SIFixWWMLiveness.cpp
SIFoldOperands.cpp		SIFoldOperands.cpp
SIFrameLowering.cpp		SIFrameLowering.cpp
SIInsertSkips.cpp		SIInsertSkips.cpp
SIInsertWaitcnts.cpp		SIInsertWaitcnts.cpp
SIInsertWaits.cpp
SIInstrInfo.cpp		SIInstrInfo.cpp
SIISelLowering.cpp		SIISelLowering.cpp
SILoadStoreOptimizer.cpp		SILoadStoreOptimizer.cpp
SILowerControlFlow.cpp		SILowerControlFlow.cpp
SILowerI1Copies.cpp		SILowerI1Copies.cpp
SIMachineFunctionInfo.cpp		SIMachineFunctionInfo.cpp
SIMachineScheduler.cpp		SIMachineScheduler.cpp
SIMemoryLegalizer.cpp		SIMemoryLegalizer.cpp
Show All 15 Lines

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

Show First 20 Lines • Show All 380 Lines • ▼ Show 20 Lines	DenseMap<MachineBasicBlock *, std::unique_ptr<BlockWaitcntBrackets>>
BlockWaitcntBracketsMap;		BlockWaitcntBracketsMap;

std::vector<MachineBasicBlock *> BlockWaitcntProcessedSet;		std::vector<MachineBasicBlock *> BlockWaitcntProcessedSet;

DenseMap<MachineLoop *, std::unique_ptr<LoopWaitcntData>> LoopWaitcntDataMap;		DenseMap<MachineLoop *, std::unique_ptr<LoopWaitcntData>> LoopWaitcntDataMap;

std::vector<std::unique_ptr<BlockWaitcntBrackets>> KillWaitBrackets;		std::vector<std::unique_ptr<BlockWaitcntBrackets>> KillWaitBrackets;

bool ForceEmitZeroWaitcnt;		// ForceEmitZeroWaitcnts: force all waitcnts insts to be s_waitcnt 0
		// because of amdgpu-waitcnt-forcezero flag
		bool ForceEmitZeroWaitcnts;
bool ForceEmitWaitcnt[NUM_INST_CNTS];		bool ForceEmitWaitcnt[NUM_INST_CNTS];

public:		public:
static char ID;		static char ID;

SIInsertWaitcnts() : MachineFunctionPass(ID) {}		SIInsertWaitcnts() : MachineFunctionPass(ID) {}

bool runOnMachineFunction(MachineFunction &MF) override;		bool runOnMachineFunction(MachineFunction &MF) override;
▲ Show 20 Lines • Show All 478 Lines • ▼ Show 20 Lines
/// and if so what the value of each counter is.		/// and if so what the value of each counter is.
/// The "score bracket" is bound by the lower bound and upper bound		/// The "score bracket" is bound by the lower bound and upper bound
/// scores (_score_LB and _score_ub respectively).		/// scores (_score_LB and _score_ub respectively).
void SIInsertWaitcnts::generateWaitcntInstBefore(		void SIInsertWaitcnts::generateWaitcntInstBefore(
MachineInstr &MI, BlockWaitcntBrackets *ScoreBrackets) {		MachineInstr &MI, BlockWaitcntBrackets *ScoreBrackets) {
// To emit, or not to emit - that's the question!		// To emit, or not to emit - that's the question!
// Start with an assumption that there is no need to emit.		// Start with an assumption that there is no need to emit.
unsigned int EmitWaitcnt = 0;		unsigned int EmitWaitcnt = 0;

// No need to wait before phi. If a phi-move exists, then the wait should		// No need to wait before phi. If a phi-move exists, then the wait should
// has been inserted before the move. If a phi-move does not exist, then		// has been inserted before the move. If a phi-move does not exist, then
// wait should be inserted before the real use. The same is true for		// wait should be inserted before the real use. The same is true for
// sc-merge. It is not a coincident that all these cases correspond to the		// sc-merge. It is not a coincident that all these cases correspond to the
// instructions that are skipped in the assembling loop.		// instructions that are skipped in the assembling loop.
bool NeedLineMapping = false; // TODO: Check on this.		bool NeedLineMapping = false; // TODO: Check on this.
setForceEmitWaitcnt();

		// ForceEmitZeroWaitcnt: force a single s_waitcnt 0 due to hw bug
		bool ForceEmitZeroWaitcnt = false;

		setForceEmitWaitcnt();
bool IsForceEmitWaitcnt = isForceEmitWaitcnt();		bool IsForceEmitWaitcnt = isForceEmitWaitcnt();

if (MI.isDebugValue() &&		if (MI.isDebugValue() &&
// TODO: any other opcode?		// TODO: any other opcode?
!NeedLineMapping) {		!NeedLineMapping) {
return;		return;
}		}

▲ Show 20 Lines • Show All 223 Lines • ▼ Show 20 Lines	if (readsVCCZ(MI) && ST->getGeneration() <= SISubtarget::SEA_ISLANDS) {
if (ScoreBrackets->getScoreLB(LGKM_CNT) <		if (ScoreBrackets->getScoreLB(LGKM_CNT) <
ScoreBrackets->getScoreUB(LGKM_CNT) &&		ScoreBrackets->getScoreUB(LGKM_CNT) &&
ScoreBrackets->hasPendingSMEM()) {		ScoreBrackets->hasPendingSMEM()) {
// Wait on everything, not just LGKM. vccz reads usually come from		// Wait on everything, not just LGKM. vccz reads usually come from
// terminators, and we always wait on everything at the end of the		// terminators, and we always wait on everything at the end of the
// block, so if we only wait on LGKM here, we might end up with		// block, so if we only wait on LGKM here, we might end up with
// another s_waitcnt inserted right after this if there are non-LGKM		// another s_waitcnt inserted right after this if there are non-LGKM
// instructions still outstanding.		// instructions still outstanding.
		// FIXME: this is too conservative / the comment is wrong.
		// We don't wait on everything at the end of the block and we combine
		// waitcnts so we should never have back-to-back waitcnts.
ForceEmitZeroWaitcnt = true;		ForceEmitZeroWaitcnt = true;
EmitWaitcnt = true;		EmitWaitcnt = true;
}		}
}		}

// Does this operand processing indicate s_wait counter update?		// Does this operand processing indicate s_wait counter update?
if (EmitWaitcnt \|\| IsForceEmitWaitcnt) {		if (EmitWaitcnt \|\| IsForceEmitWaitcnt) {
int CntVal[NUM_INST_CNTS];		int CntVal[NUM_INST_CNTS];

bool UseDefaultWaitcntStrategy = true;		bool UseDefaultWaitcntStrategy = true;
if (ForceEmitZeroWaitcnt) {		if (ForceEmitZeroWaitcnt \|\| ForceEmitZeroWaitcnts) {
// Force all waitcnts to 0.		// Force all waitcnts to 0.
for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;		for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
T = (enum InstCounterType)(T + 1)) {		T = (enum InstCounterType)(T + 1)) {
ScoreBrackets->setScoreLB(T, ScoreBrackets->getScoreUB(T));		ScoreBrackets->setScoreLB(T, ScoreBrackets->getScoreUB(T));
}		}
CntVal[VM_CNT] = 0;		CntVal[VM_CNT] = 0;
CntVal[EXP_CNT] = 0;		CntVal[EXP_CNT] = 0;
CntVal[LGKM_CNT] = 0;		CntVal[LGKM_CNT] = 0;
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines	if (EmitWaitcnt != 0 \|\| IsForceEmitWaitcnt) {
}		}
break;		break;
// TODO: skip over instructions which never require wait.		// TODO: skip over instructions which never require wait.
}		}
break;		break;
}		}
if (insertSWaitInst) {		if (insertSWaitInst) {
if (OldWaitcnt && OldWaitcnt->getOpcode() == AMDGPU::S_WAITCNT) {		if (OldWaitcnt && OldWaitcnt->getOpcode() == AMDGPU::S_WAITCNT) {
if (ForceEmitZeroWaitcnt)		if (ForceEmitZeroWaitcnts)
DEBUG(dbgs() << "Force emit s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)\n");		DEBUG(dbgs() << "Force emit s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)\n");
if (IsForceEmitWaitcnt)		if (IsForceEmitWaitcnt)
DEBUG(dbgs() << "Force emit a s_waitcnt due to debug counter\n");		DEBUG(dbgs() << "Force emit a s_waitcnt due to debug counter\n");

OldWaitcnt->getOperand(0).setImm(Enc);		OldWaitcnt->getOperand(0).setImm(Enc);
if (!OldWaitcnt->getParent())		if (!OldWaitcnt->getParent())
MI.getParent()->insert(MI, OldWaitcnt);		MI.getParent()->insert(MI, OldWaitcnt);

▲ Show 20 Lines • Show All 579 Lines • ▼ Show 20 Lines	bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
TII = ST->getInstrInfo();		TII = ST->getInstrInfo();
TRI = &TII->getRegisterInfo();		TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo();		MRI = &MF.getRegInfo();
MLI = &getAnalysis<MachineLoopInfo>();		MLI = &getAnalysis<MachineLoopInfo>();
IV = AMDGPU::IsaInfo::getIsaVersion(ST->getFeatureBits());		IV = AMDGPU::IsaInfo::getIsaVersion(ST->getFeatureBits());
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();		const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
AMDGPUASI = ST->getAMDGPUAS();		AMDGPUASI = ST->getAMDGPUAS();

ForceEmitZeroWaitcnt = ForceEmitZeroFlag;		ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;		for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
T = (enum InstCounterType)(T + 1))		T = (enum InstCounterType)(T + 1))
ForceEmitWaitcnt[T] = false;		ForceEmitWaitcnt[T] = false;

HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV);		HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV);
HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);		HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);		HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV);

▲ Show 20 Lines • Show All 164 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIInsertWaits.cpp

This file was deleted.

	//===- SILowerControlFlow.cpp - Use predicates for control flow -----------===//
	//
	// The LLVM Compiler Infrastructure
	//
	// This file is distributed under the University of Illinois Open Source
	// License. See LICENSE.TXT for details.
	//
	//===----------------------------------------------------------------------===//
	//
	/// \file
	/// Insert wait instructions for memory reads and writes.
	///
	/// Memory reads and writes are issued asynchronously, so we need to insert
	/// S_WAITCNT instructions when we want to access any of their results or
	/// overwrite any register that's used asynchronously.
	//
	//===----------------------------------------------------------------------===//

	#include "AMDGPU.h"
	#include "AMDGPUSubtarget.h"
	#include "SIDefines.h"
	#include "SIInstrInfo.h"
	#include "SIMachineFunctionInfo.h"
	#include "SIRegisterInfo.h"
	#include "Utils/AMDGPUBaseInfo.h"
	#include "llvm/ADT/SmallVector.h"
	#include "llvm/ADT/StringRef.h"
	#include "llvm/CodeGen/MachineBasicBlock.h"
	#include "llvm/CodeGen/MachineFunction.h"
	#include "llvm/CodeGen/MachineFunctionPass.h"
	#include "llvm/CodeGen/MachineInstr.h"
	#include "llvm/CodeGen/MachineInstrBuilder.h"
	#include "llvm/CodeGen/MachineOperand.h"
	#include "llvm/CodeGen/MachineRegisterInfo.h"
	#include "llvm/IR/DebugLoc.h"
	#include "llvm/MC/MCInstrDesc.h"
	#include "llvm/Pass.h"
	#include "llvm/Support/Debug.h"
	#include "llvm/Support/raw_ostream.h"
	#include <algorithm>
	#include <cassert>
	#include <cstdint>
	#include <cstring>
	#include <utility>

	#define DEBUG_TYPE "si-insert-waits"

	using namespace llvm;

	namespace {

	/// One variable for each of the hardware counters
	using Counters = union {
	struct {
	unsigned VM;
	unsigned EXP;
	unsigned LGKM;
	} Named;
	unsigned Array[3];
	};

	using InstType = enum {
	OTHER,
	SMEM,
	VMEM
	};

	using RegCounters = Counters[512];
	using RegInterval = std::pair<unsigned, unsigned>;

	class SIInsertWaits : public MachineFunctionPass {
	private:
	const SISubtarget *ST = nullptr;
	const SIInstrInfo *TII = nullptr;
	const SIRegisterInfo *TRI = nullptr;
	const MachineRegisterInfo *MRI;
	AMDGPU::IsaInfo::IsaVersion ISA;

	/// Constant zero value
	static const Counters ZeroCounts;

	/// Hardware limits
	Counters HardwareLimits;

	/// Counter values we have already waited on.
	Counters WaitedOn;

	/// Counter values that we must wait on before the next counter
	/// increase.
	Counters DelayedWaitOn;

	/// Counter values for last instruction issued.
	Counters LastIssued;

	/// Registers used by async instructions.
	RegCounters UsedRegs;

	/// Registers defined by async instructions.
	RegCounters DefinedRegs;

	/// Different export instruction types seen since last wait.
	unsigned ExpInstrTypesSeen = 0;

	/// Type of the last opcode.
	InstType LastOpcodeType;

	bool LastInstWritesM0;

	/// Whether or not we have flat operations outstanding.
	bool IsFlatOutstanding;

	/// Whether the machine function returns void
	bool ReturnsVoid;

	/// Whether the VCCZ bit is possibly corrupt
	bool VCCZCorrupt = false;

	/// Get increment/decrement amount for this instruction.
	Counters getHwCounts(MachineInstr &MI);

	/// Is operand relevant for async execution?
	bool isOpRelevant(MachineOperand &Op);

	/// Get register interval an operand affects.
	RegInterval getRegInterval(const TargetRegisterClass *RC,
	const MachineOperand &Reg) const;

	/// Handle instructions async components
	void pushInstruction(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator I,
	const Counters& Increment);

	/// Insert the actual wait instruction
	bool insertWait(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator I,
	const Counters &Counts);

	/// Handle existing wait instructions (from intrinsics)
	void handleExistingWait(MachineBasicBlock::iterator I);

	/// Do we need def2def checks?
	bool unorderedDefines(MachineInstr &MI);

	/// Resolve all operand dependencies to counter requirements
	Counters handleOperands(MachineInstr &MI);

	/// Insert S_NOP between an instruction writing M0 and S_SENDMSG.
	void handleSendMsg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I);

	/// Return true if there are LGKM instrucitons that haven't been waited on
	/// yet.
	bool hasOutstandingLGKM() const;

	public:
	static char ID;

	SIInsertWaits() : MachineFunctionPass(ID) {}

	bool runOnMachineFunction(MachineFunction &MF) override;

	StringRef getPassName() const override {
	return "SI insert wait instructions";
	}

	void getAnalysisUsage(AnalysisUsage &AU) const override {
	AU.setPreservesCFG();
	MachineFunctionPass::getAnalysisUsage(AU);
	}
	};

	} // end anonymous namespace

	INITIALIZE_PASS_BEGIN(SIInsertWaits, DEBUG_TYPE,
	"SI Insert Waits", false, false)
	INITIALIZE_PASS_END(SIInsertWaits, DEBUG_TYPE,
	"SI Insert Waits", false, false)

	char SIInsertWaits::ID = 0;

	char &llvm::SIInsertWaitsID = SIInsertWaits::ID;

	FunctionPass *llvm::createSIInsertWaitsPass() {
	return new SIInsertWaits();
	}

	const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };

	static bool readsVCCZ(const MachineInstr &MI) {
	unsigned Opc = MI.getOpcode();
	return (Opc == AMDGPU::S_CBRANCH_VCCNZ \|\| Opc == AMDGPU::S_CBRANCH_VCCZ) &&
	!MI.getOperand(1).isUndef();
	}

	bool SIInsertWaits::hasOutstandingLGKM() const {
	return WaitedOn.Named.LGKM != LastIssued.Named.LGKM;
	}

	Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
	uint64_t TSFlags = MI.getDesc().TSFlags;
	Counters Result = { { 0, 0, 0 } };

	Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT);

	// Only consider stores or EXP for EXP_CNT
	Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT) && MI.mayStore();

	// LGKM may uses larger values
	if (TSFlags & SIInstrFlags::LGKM_CNT) {

	if (TII->isSMRD(MI)) {

	if (MI.getNumOperands() != 0) {
	assert(MI.getOperand(0).isReg() &&
	"First LGKM operand must be a register!");

	// XXX - What if this is a write into a super register?
	const TargetRegisterClass *RC = TII->getOpRegClass(MI, 0);
	unsigned Size = TRI->getRegSizeInBits(*RC);
	Result.Named.LGKM = Size > 32 ? 2 : 1;
	} else {
	// s_dcache_inv etc. do not have a destination register. Assume we
	// want a wait on these.
	// XXX - What is the right value?
	Result.Named.LGKM = 1;
	}
	} else {
	// DS
	Result.Named.LGKM = 1;
	}

	} else {
	Result.Named.LGKM = 0;
	}

	return Result;
	}

	bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
	// Constants are always irrelevant
	if (!Op.isReg() \|\| !TRI->isInAllocatableClass(Op.getReg()))
	return false;

	// Defines are always relevant
	if (Op.isDef())
	return true;

	// For exports all registers are relevant.
	// TODO: Skip undef/disabled registers.
	MachineInstr &MI = *Op.getParent();
	if (TII->isEXP(MI))
	return true;

	// For stores the stored value is also relevant
	if (!MI.getDesc().mayStore())
	return false;

	// Check if this operand is the value being stored.
	// Special case for DS/FLAT instructions, since the address
	// operand comes before the value operand and it may have
	// multiple data operands.

	if (TII->isDS(MI)) {
	MachineOperand *Data0 = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
	if (Data0 && Op.isIdenticalTo(*Data0))
	return true;

	MachineOperand *Data1 = TII->getNamedOperand(MI, AMDGPU::OpName::data1);
	return Data1 && Op.isIdenticalTo(*Data1);
	}

	if (TII->isFLAT(MI)) {
	MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::vdata);
	if (Data && Op.isIdenticalTo(*Data))
	return true;
	}

	// NOTE: This assumes that the value operand is before the
	// address operand, and that there is only one value operand.
	for (MachineInstr::mop_iterator I = MI.operands_begin(),
	E = MI.operands_end(); I != E; ++I) {

	if (I->isReg() && I->isUse())
	return Op.isIdenticalTo(*I);
	}

	return false;
	}

	RegInterval SIInsertWaits::getRegInterval(const TargetRegisterClass *RC,
	const MachineOperand &Reg) const {
	unsigned Size = TRI->getRegSizeInBits(*RC);
	assert(Size >= 32);

	RegInterval Result;
	Result.first = TRI->getEncodingValue(Reg.getReg());
	Result.second = Result.first + Size / 32;

	return Result;
	}

	void SIInsertWaits::pushInstruction(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator I,
	const Counters &Increment) {
	// Get the hardware counter increments and sum them up
	Counters Limit = ZeroCounts;
	unsigned Sum = 0;

	if (TII->mayAccessFlatAddressSpace(*I))
	IsFlatOutstanding = true;

	for (unsigned i = 0; i < 3; ++i) {
	LastIssued.Array[i] += Increment.Array[i];
	if (Increment.Array[i])
	Limit.Array[i] = LastIssued.Array[i];
	Sum += Increment.Array[i];
	}

	// If we don't increase anything then that's it
	if (Sum == 0) {
	LastOpcodeType = OTHER;
	return;
	}

	if (ST->getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) {
	// Any occurrence of consecutive VMEM or SMEM instructions forms a VMEM
	// or SMEM clause, respectively.
	//
	// The temporary workaround is to break the clauses with S_NOP.
	//
	// The proper solution would be to allocate registers such that all source
	// and destination registers don't overlap, e.g. this is illegal:
	// r0 = load r2
	// r2 = load r0
	if (LastOpcodeType == VMEM && Increment.Named.VM) {
	// Insert a NOP to break the clause.
	BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP))
	.addImm(0);
	LastInstWritesM0 = false;
	}

	if (TII->isSMRD(*I))
	LastOpcodeType = SMEM;
	else if (Increment.Named.VM)
	LastOpcodeType = VMEM;
	}

	// Remember which export instructions we have seen
	if (Increment.Named.EXP) {
	ExpInstrTypesSeen \|= TII->isEXP(*I) ? 1 : 2;
	}

	for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
	MachineOperand &Op = I->getOperand(i);
	if (!isOpRelevant(Op))
	continue;

	const TargetRegisterClass RC = TII->getOpRegClass(I, i);
	RegInterval Interval = getRegInterval(RC, Op);
	for (unsigned j = Interval.first; j < Interval.second; ++j) {

	// Remember which registers we define
	if (Op.isDef())
	DefinedRegs[j] = Limit;

	// and which one we are using
	if (Op.isUse())
	UsedRegs[j] = Limit;
	}
	}
	}

	bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator I,
	const Counters &Required) {
	// End of program? No need to wait on anything
	// A function not returning void needs to wait, because other bytecode will
	// be appended after it and we don't know what it will be.
	if (I != MBB.end() && I->getOpcode() == AMDGPU::S_ENDPGM && ReturnsVoid)
	return false;

	// Figure out if the async instructions execute in order
	bool Ordered[3];

	// VM_CNT is always ordered except when there are flat instructions, which
	// can return out of order.
	Ordered[0] = !IsFlatOutstanding;

	// EXP_CNT is unordered if we have both EXP & VM-writes
	Ordered[1] = ExpInstrTypesSeen == 3;

	// LGKM_CNT is handled as always unordered. TODO: Handle LDS and GDS
	Ordered[2] = false;

	// The values we are going to put into the S_WAITCNT instruction
	Counters Counts = HardwareLimits;

	// Do we really need to wait?
	bool NeedWait = false;

	for (unsigned i = 0; i < 3; ++i) {
	if (Required.Array[i] <= WaitedOn.Array[i])
	continue;

	NeedWait = true;

	if (Ordered[i]) {
	unsigned Value = LastIssued.Array[i] - Required.Array[i];

	// Adjust the value to the real hardware possibilities.
	Counts.Array[i] = std::min(Value, HardwareLimits.Array[i]);
	} else
	Counts.Array[i] = 0;

	// Remember on what we have waited on.
	WaitedOn.Array[i] = LastIssued.Array[i] - Counts.Array[i];
	}

	if (!NeedWait)
	return false;

	// Reset EXP_CNT instruction types
	if (Counts.Named.EXP == 0)
	ExpInstrTypesSeen = 0;

	// Build the wait instruction
	BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
	.addImm(AMDGPU::encodeWaitcnt(ISA,
	Counts.Named.VM,
	Counts.Named.EXP,
	Counts.Named.LGKM));

	LastOpcodeType = OTHER;
	LastInstWritesM0 = false;
	IsFlatOutstanding = false;
	return true;
	}

	/// helper function for handleOperands
	static void increaseCounters(Counters &Dst, const Counters &Src) {
	for (unsigned i = 0; i < 3; ++i)
	Dst.Array[i] = std::max(Dst.Array[i], Src.Array[i]);
	}

	/// check whether any of the counters is non-zero
	static bool countersNonZero(const Counters &Counter) {
	for (unsigned i = 0; i < 3; ++i)
	if (Counter.Array[i])
	return true;
	return false;
	}

	void SIInsertWaits::handleExistingWait(MachineBasicBlock::iterator I) {
	assert(I->getOpcode() == AMDGPU::S_WAITCNT);

	unsigned Imm = I->getOperand(0).getImm();
	Counters Counts, WaitOn;

	Counts.Named.VM = AMDGPU::decodeVmcnt(ISA, Imm);
	Counts.Named.EXP = AMDGPU::decodeExpcnt(ISA, Imm);
	Counts.Named.LGKM = AMDGPU::decodeLgkmcnt(ISA, Imm);

	for (unsigned i = 0; i < 3; ++i) {
	if (Counts.Array[i] <= LastIssued.Array[i])
	WaitOn.Array[i] = LastIssued.Array[i] - Counts.Array[i];
	else
	WaitOn.Array[i] = 0;
	}

	increaseCounters(DelayedWaitOn, WaitOn);
	}

	Counters SIInsertWaits::handleOperands(MachineInstr &MI) {
	Counters Result = ZeroCounts;

	// For each register affected by this instruction increase the result
	// sequence.
	//
	// TODO: We could probably just look at explicit operands if we removed VCC /
	// EXEC from SMRD dest reg classes.
	for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
	MachineOperand &Op = MI.getOperand(i);
	if (!Op.isReg() \|\| !TRI->isInAllocatableClass(Op.getReg()))
	continue;

	const TargetRegisterClass *RC = TII->getOpRegClass(MI, i);
	RegInterval Interval = getRegInterval(RC, Op);
	for (unsigned j = Interval.first; j < Interval.second; ++j) {
	if (Op.isDef()) {
	increaseCounters(Result, UsedRegs[j]);
	increaseCounters(Result, DefinedRegs[j]);
	}

	if (Op.isUse())
	increaseCounters(Result, DefinedRegs[j]);
	}
	}

	return Result;
	}

	void SIInsertWaits::handleSendMsg(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator I) {
	if (ST->getGeneration() < SISubtarget::VOLCANIC_ISLANDS)
	return;

	// There must be "S_NOP 0" between an instruction writing M0 and S_SENDMSG.
	if (LastInstWritesM0 && (I->getOpcode() == AMDGPU::S_SENDMSG \|\| I->getOpcode() == AMDGPU::S_SENDMSGHALT)) {
	BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_NOP)).addImm(0);
	LastInstWritesM0 = false;
	return;
	}

	// Set whether this instruction sets M0
	LastInstWritesM0 = false;

	unsigned NumOperands = I->getNumOperands();
	for (unsigned i = 0; i < NumOperands; i++) {
	const MachineOperand &Op = I->getOperand(i);

	if (Op.isReg() && Op.isDef() && Op.getReg() == AMDGPU::M0)
	LastInstWritesM0 = true;
	}
	}

	/// Return true if \p MBB has one successor immediately following, and is its
	/// only predecessor
	static bool hasTrivialSuccessor(const MachineBasicBlock &MBB) {
	if (MBB.succ_size() != 1)
	return false;

	const MachineBasicBlock Succ = MBB.succ_begin();
	return (Succ->pred_size() == 1) && MBB.isLayoutSuccessor(Succ);
	}

	// FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States"
	// around other non-memory instructions.
	bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
	bool Changes = false;

	ST = &MF.getSubtarget<SISubtarget>();
	TII = ST->getInstrInfo();
	TRI = &TII->getRegisterInfo();
	MRI = &MF.getRegInfo();
	ISA = AMDGPU::IsaInfo::getIsaVersion(ST->getFeatureBits());
	const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

	HardwareLimits.Named.VM = AMDGPU::getVmcntBitMask(ISA);
	HardwareLimits.Named.EXP = AMDGPU::getExpcntBitMask(ISA);
	HardwareLimits.Named.LGKM = AMDGPU::getLgkmcntBitMask(ISA);

	WaitedOn = ZeroCounts;
	DelayedWaitOn = ZeroCounts;
	LastIssued = ZeroCounts;
	LastOpcodeType = OTHER;
	LastInstWritesM0 = false;
	IsFlatOutstanding = false;
	ReturnsVoid = MFI->returnsVoid();

	memset(&UsedRegs, 0, sizeof(UsedRegs));
	memset(&DefinedRegs, 0, sizeof(DefinedRegs));

	SmallVector<MachineInstr *, 4> RemoveMI;
	SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;

	bool HaveScalarStores = false;

	for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
	BI != BE; ++BI) {
	MachineBasicBlock &MBB = *BI;

	for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
	I != E; ++I) {
	if (!HaveScalarStores && TII->isScalarStore(*I))
	HaveScalarStores = true;

	if (ST->getGeneration() <= SISubtarget::SEA_ISLANDS) {
	// There is a hardware bug on CI/SI where SMRD instruction may corrupt
	// vccz bit, so when we detect that an instruction may read from a
	// corrupt vccz bit, we need to:
	// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD operations to
	// complete.
	// 2. Restore the correct value of vccz by writing the current value
	// of vcc back to vcc.

	if (TII->isSMRD(I->getOpcode())) {
	VCCZCorrupt = true;
	} else if (!hasOutstandingLGKM() && I->modifiesRegister(AMDGPU::VCC, TRI)) {
	// FIXME: We only care about SMRD instructions here, not LDS or GDS.
	// Whenever we store a value in vcc, the correct value of vccz is
	// restored.
	VCCZCorrupt = false;
	}

	// Check if we need to apply the bug work-around
	if (VCCZCorrupt && readsVCCZ(*I)) {
	DEBUG(dbgs() << "Inserting vccz bug work-around before: " << *I << '\n');

	// Wait on everything, not just LGKM. vccz reads usually come from
	// terminators, and we always wait on everything at the end of the
	// block, so if we only wait on LGKM here, we might end up with
	// another s_waitcnt inserted right after this if there are non-LGKM
	// instructions still outstanding.
	insertWait(MBB, I, LastIssued);

	// Restore the vccz bit. Any time a value is written to vcc, the vcc
	// bit is updated, so we can restore the bit by reading the value of
	// vcc and then writing it back to the register.
	BuildMI(MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
	AMDGPU::VCC)
	.addReg(AMDGPU::VCC);
	}
	}

	// Record pre-existing, explicitly requested waits
	if (I->getOpcode() == AMDGPU::S_WAITCNT) {
	handleExistingWait(*I);
	RemoveMI.push_back(&*I);
	continue;
	}

	Counters Required;

	// Wait for everything before a barrier.
	//
	// S_SENDMSG implicitly waits for all outstanding LGKM transfers to finish,
	// but we also want to wait for any other outstanding transfers before
	// signalling other hardware blocks
	if ((I->getOpcode() == AMDGPU::S_BARRIER &&
	!ST->hasAutoWaitcntBeforeBarrier()) \|\|
	I->getOpcode() == AMDGPU::S_SENDMSG \|\|
	I->getOpcode() == AMDGPU::S_SENDMSGHALT)
	Required = LastIssued;
	else
	Required = handleOperands(*I);

	Counters Increment = getHwCounts(*I);

	if (countersNonZero(Required) \|\| countersNonZero(Increment))
	increaseCounters(Required, DelayedWaitOn);

	Changes \|= insertWait(MBB, I, Required);

	pushInstruction(MBB, I, Increment);
	handleSendMsg(MBB, I);

	if (I->getOpcode() == AMDGPU::S_ENDPGM \|\|
	I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
	EndPgmBlocks.push_back(&MBB);
	}

	// Wait for everything at the end of the MBB. If there is only one
	// successor, we can defer this until the uses there.
	if (!hasTrivialSuccessor(MBB))
	Changes \|= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);
	}

	if (HaveScalarStores) {
	// If scalar writes are used, the cache must be flushed or else the next
	// wave to reuse the same scratch memory can be clobbered.
	//
	// Insert s_dcache_wb at wave termination points if there were any scalar
	// stores, and only if the cache hasn't already been flushed. This could be
	// improved by looking across blocks for flushes in postdominating blocks
	// from the stores but an explicitly requested flush is probably very rare.
	for (MachineBasicBlock *MBB : EndPgmBlocks) {
	bool SeenDCacheWB = false;

	for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
	I != E; ++I) {
	if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
	SeenDCacheWB = true;
	else if (TII->isScalarStore(*I))
	SeenDCacheWB = false;

	// FIXME: It would be better to insert this before a waitcnt if any.
	if ((I->getOpcode() == AMDGPU::S_ENDPGM \|\|
	I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && !SeenDCacheWB) {
	Changes = true;
	BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB));
	}
	}
	}
	}

	for (MachineInstr *I : RemoveMI)
	I->eraseFromParent();

	if (!MFI->isEntryFunction()) {
	// Wait for any outstanding memory operations that the input registers may
	// depend on. We can't track them and it's better to the wait after the
	// costly call sequence.

	// TODO: Could insert earlier and schedule more liberally with operations
	// that only use caller preserved registers.
	MachineBasicBlock &EntryBB = MF.front();
	BuildMI(EntryBB, EntryBB.getFirstNonPHI(), DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
	.addImm(0);

	Changes = true;
	}

	return Changes;
	}

test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir

# RUN: llc -run-pass si-insert-waits -march=amdgcn -mcpu=tahiti -o - %s \| FileCheck %s		# RUN: llc -run-pass si-insert-waitcnts -march=amdgcn -mcpu=tahiti -o - %s \| FileCheck %s
--- \|		--- \|

define amdgpu_kernel void @vccz_corrupt_workaround(float %cond, i32 addrspace(1)* %out) #0 {		define amdgpu_kernel void @vccz_corrupt_workaround(float %cond, i32 addrspace(1)* %out) #0 {
entry:		entry:
%cmp0 = fcmp oeq float %cond, 0.000000e+00		%cmp0 = fcmp oeq float %cond, 0.000000e+00
br i1 %cmp0, label %if, label %else, !structurizecfg.uniform !0, !amdgpu.uniform !0		br i1 %cmp0, label %if, label %else, !structurizecfg.uniform !0, !amdgpu.uniform !0

else: ; preds = %entry		else: ; preds = %entry
Show All 32 Lines	--- \|
attributes #1 = { readnone }		attributes #1 = { readnone }

!0 = !{}		!0 = !{}

...		...
---		---
# CHECK-LABEL: name: vccz_corrupt_workaround		# CHECK-LABEL: name: vccz_corrupt_workaround
# CHECK: $vcc = V_CMP_EQ_F32		# CHECK: $vcc = V_CMP_EQ_F32
		# CHECK-NEXT: S_WAITCNT 0
# CHECK-NEXT: $vcc = S_MOV_B64 $vcc		# CHECK-NEXT: $vcc = S_MOV_B64 $vcc
# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2, implicit killed $vcc		# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2, implicit killed $vcc

name: vccz_corrupt_workaround		name: vccz_corrupt_workaround
alignment: 0		alignment: 0
exposesReturnsTwice: false		exposesReturnsTwice: false
legalized: false		legalized: false
regBankSelected: false		regBankSelected: false
Show All 12 Lines	frameInfo:
adjustsStack: false		adjustsStack: false
hasCalls: false		hasCalls: false
maxCallFrameSize: 0		maxCallFrameSize: 0
hasOpaqueSPAdjustment: false		hasOpaqueSPAdjustment: false
hasVAStart: false		hasVAStart: false
hasMustTailInVarArgFunc: false		hasMustTailInVarArgFunc: false
body: \|		body: \|
bb.0.entry:		bb.0.entry:
liveins: $sgpr0_sgpr1		liveins: $sgpr0_sgpr1, $vcc

$sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 9, 0 :: (non-temporal dereferenceable invariant load 4 from `float addrspace(2)* undef`)
$sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)		$sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed $sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
$sgpr7 = S_MOV_B32 61440		$sgpr7 = S_MOV_B32 61440
$sgpr6 = S_MOV_B32 -1		$sgpr6 = S_MOV_B32 -1
$vcc = V_CMP_EQ_F32_e64 0, 0, 0, $sgpr2, 0, implicit $exec		$vcc = V_CMP_EQ_F32_e64 0, 0, 0, undef $sgpr2, 0, implicit $exec
S_CBRANCH_VCCZ %bb.1, implicit killed $vcc		S_CBRANCH_VCCZ %bb.1, implicit killed $vcc

bb.2.if:		bb.2.if:
liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003		liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003

$vgpr0 = V_MOV_B32_e32 9, implicit $exec		$vgpr0 = V_MOV_B32_e32 9, implicit $exec
BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)		BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
$vgpr0 = V_MOV_B32_e32 0, implicit $exec		$vgpr0 = V_MOV_B32_e32 0, implicit $exec
Show All 12 Lines	bb.3.done:
$sgpr3 = S_MOV_B32 61440		$sgpr3 = S_MOV_B32 61440
$sgpr2 = S_MOV_B32 -1		$sgpr2 = S_MOV_B32 -1
BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)		BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
S_ENDPGM		S_ENDPGM

...		...
---		---
# CHECK-LABEL: name: vccz_corrupt_undef_vcc		# CHECK-LABEL: name: vccz_corrupt_undef_vcc
# CHECK: S_WAITCNT		# CHECK: S_WAITCNT 3855
# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef $vcc		# CHECK-NEXT: $vgpr0 = V_MOV_B32_e32

name: vccz_corrupt_undef_vcc		name: vccz_corrupt_undef_vcc
alignment: 0		alignment: 0
exposesReturnsTwice: false		exposesReturnsTwice: false
legalized: false		legalized: false
regBankSelected: false		regBankSelected: false
selected: false		selected: false
tracksRegLiveness: true		tracksRegLiveness: true
▲ Show 20 Lines • Show All 49 Lines • Show Last 20 Lines

test/CodeGen/AMDGPU/waitcnt.mir

# RUN: llc -march=amdgcn -mcpu=fiji -run-pass si-insert-waits %s -o - \| FileCheck %s		# RUN: llc -march=amdgcn -mcpu=fiji -run-pass si-insert-waitcnts %s -o - \| FileCheck %s

--- \|		--- \|
define amdgpu_kernel void @flat_zero_waitcnt(i32 addrspace(1)* %global4,		define amdgpu_kernel void @flat_zero_waitcnt(i32 addrspace(1)* %global4,
<4 x i32> addrspace(1)* %global16,		<4 x i32> addrspace(1)* %global16,
i32* %flat4,		i32* %flat4,
<4 x i32>* %flat16) {		<4 x i32>* %flat16) {
ret void		ret void
}		}
Show All 10 Lines
---		---

# CHECK-LABEL: name: flat_zero_waitcnt		# CHECK-LABEL: name: flat_zero_waitcnt

# CHECK-LABEL: bb.0:		# CHECK-LABEL: bb.0:
# CHECK: FLAT_LOAD_DWORD		# CHECK: FLAT_LOAD_DWORD
# CHECK: FLAT_LOAD_DWORDX4		# CHECK: FLAT_LOAD_DWORDX4
# Global loads will return in order so we should:		# Global loads will return in order so we should:
# s_waitcnt vmcnt(1) lgkmcnt(0)		# s_waitcnt vmcnt(1) lgkmcnt(1)
# CHECK-NEXT: S_WAITCNT 113		# CHECK-NEXT: S_WAITCNT 369

# CHECK-LABEL: bb.1:		# CHECK-LABEL: bb.1:
# CHECK: FLAT_LOAD_DWORD		# CHECK: FLAT_LOAD_DWORD
# CHECK: S_WAITCNT 3952		# CHECK: S_WAITCNT 368
# CHECK: FLAT_LOAD_DWORDX4		# CHECK: FLAT_LOAD_DWORDX4
# The first load has no mem operand, so we should assume it accesses the flat		# The first load has no mem operand, so we should assume it accesses the flat
# address space.		# address space.
# s_waitcnt vmcnt(0) lgkmcnt(0)		# s_waitcnt lgkmcnt(1)
# CHECK-NEXT: S_WAITCNT 127		# CHECK-NEXT: S_WAITCNT 383

# CHECK-LABEL: bb.2:		# CHECK-LABEL: bb.2:
# CHECK: FLAT_LOAD_DWORD		# CHECK: FLAT_LOAD_DWORD
# CHECK: S_WAITCNT 3952		# CHECK: S_WAITCNT 368
# CHECK: FLAT_LOAD_DWORDX4		# CHECK: FLAT_LOAD_DWORDX4

# One outstand loads access the flat address space.		# One outstanding load accesses the flat address space.
# s_waitcnt vmcnt(0) lgkmcnt(0)		# s_waitcnt lgkmcnt(1)
# CHECK-NEXT: S_WAITCNT 127		# CHECK-NEXT: S_WAITCNT 383

name: flat_zero_waitcnt		name: flat_zero_waitcnt

body: \|		body: \|
bb.0:		bb.0:
successors: %bb.1		successors: %bb.1
$vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.global4)		$vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.global4)
$vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.global16)		$vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 $vgpr7_vgpr8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16 from %ir.global16)
Show All 38 Lines	bb.1:
S_ENDPGM		S_ENDPGM
...		...
---		---
# The block has a single predecessor with a single successor, but it		# The block has a single predecessor with a single successor, but it
# is not the next block so it's non-obvious that the wait is not needed.		# is not the next block so it's non-obvious that the wait is not needed.


# CHECK-LABEL: name: single_branch_successor_not_next_block		# CHECK-LABEL: name: single_branch_successor_not_next_block
# CHECK: $vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2
# CHECK-NEXT: S_WAITCNT 112

# CHECK: bb.1		# CHECK: bb.1
# CHECK-NEXT: FLAT_STORE_DWORD		# CHECK-NEXT: FLAT_STORE_DWORD
# CHECK-NEXT: S_ENDPGM		# CHECK-NEXT: S_ENDPGM

# CHECK: bb.2:		# CHECK: bb.2:
# CHECK-NEXT: V_LSHLREV_B64		# CHECK-NEXT: V_LSHLREV_B64
		# CHECK-NEXT: S_WAITCNT 112
# CHECK-NEXT: FLAT_STORE_DWORD		# CHECK-NEXT: FLAT_STORE_DWORD
name: single_branch_successor_not_next_block		name: single_branch_successor_not_next_block

body: \|		body: \|
bb.0:		bb.0:
successors: %bb.2		successors: %bb.2
$vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr		$vgpr0 = FLAT_LOAD_DWORD $vgpr1_vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr
S_BRANCH %bb.2		S_BRANCH %bb.2
Show All 10 Lines

This is an archive of the discontinued LLVM Phabricator instance.

[AMDGPU][Waitcnt] Remove the old waitcnt pass, which is no longer maintained and getting cruftyClosedPublic

Details

Diff Detail

Event Timeline

Revision Contents

Diff 145309

lib/Target/AMDGPU/AMDGPU.h

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

lib/Target/AMDGPU/CMakeLists.txt

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

lib/Target/AMDGPU/SIInsertWaits.cpp

test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir

test/CodeGen/AMDGPU/waitcnt.mir

[AMDGPU][Waitcnt] Remove the old waitcnt pass, which is no longer maintained and getting crufty
ClosedPublic