This is an archive of the discontinued LLVM Phabricator instance.

Differential D49037

AMDGPU: Refactor Subtarget classes
ClosedPublic

Authored by tstellar on Jul 6 2018, 11:25 AM.

Details

Reviewers
arsenm
jvesely
Commits
rG5bfbae5cb180: AMDGPU: Refactor Subtarget classes
rL336851: AMDGPU: Refactor Subtarget classes
Summary

This is a follow-up to r335942.

  • Merge SISubtarget into AMDGPUSubtarget and rename to GCNSubtarget
  • Rename AMDGPUCommonSubtarget to AMDGPUSubtarget

Diff Detail

Event Timeline

tstellar created this revision.Jul 6 2018, 11:25 AM
Herald added subscribers: javed.absar, t-tye, tpr and 5 others. · View Herald TranscriptJul 6 2018, 11:25 AM
Harbormaster completed remote builds in B20119: Diff 154439.Jul 6 2018, 11:25 AM
arsenm accepted this revision.Jul 6 2018, 11:35 AM

LGTM

This revision is now accepted and ready to land.Jul 6 2018, 11:35 AM
jvesely accepted this revision.Jul 6 2018, 12:37 PM

Other than the few nits mentioned in the text, LGTM.

lib/Target/AMDGPU/AMDGPUInstructionSelector.h
42

This rename duplicates GCNSubtarget declaration.

lib/Target/AMDGPU/AMDGPUSubtarget.h
230

Just out of curiosity what's the benefit of having this enum split between GCNSUbtarget and R600Subtarget, vs merging it in AMDGPUSubtarget?

981

Is this #if 0 code here intentionally? If yes an explanatory comment would be nice.

lib/Target/AMDGPU/R600RegisterInfo.h
23

Why is this necessary in R600 file? Looks like a leftover from the time when AMDGPUSubtarget was shared parent.

lib/Target/AMDGPU/SIRegisterInfo.h
27

This rename duplicates GCNSUbtarget declaration.

tstellar marked 5 inline comments as done.Jul 11 2018, 2:06 PM
tstellar added inline comments.
lib/Target/AMDGPU/AMDGPUSubtarget.h
230

There is no benefit, so I've merged the enums back together.

Closed by commit rL336851: AMDGPU: Refactor Subtarget classes (authored by tstellar). · Explain WhyJul 11 2018, 2:06 PM
This revision was automatically updated to reflect the committed changes.
tstellar marked an inline comment as done.
Herald added a subscriber: MatzeB. · View Herald TranscriptJul 11 2018, 2:06 PM

Revision Contents

PathSize
lib/
Target/
AMDGPU/
24 lines
2 lines
2 lines
4 lines
38 lines
4 lines
16 lines
4 lines
24 lines
6 lines
10 lines
4 lines
2 lines
12 lines
8 lines
4 lines
4 lines
2 lines
4 lines
4 lines
2 lines
2 lines
12 lines
2 lines
144 lines
100 lines
4 lines
6 lines
6 lines
2 lines
4 lines
12 lines
18 lines
8 lines
4 lines
4 lines
4 lines
2 lines
2 lines
2 lines
2 lines
2 lines
4 lines
4 lines
10 lines
30 lines
6 lines
58 lines
2 lines
12 lines
4 lines
6 lines
44 lines
8 lines
4 lines
2 lines
2 lines
8 lines
18 lines
2 lines
2 lines
10 lines
6 lines
24 lines
2 lines
2 lines

Diff 154439

lib/Target/AMDGPU/AMDGPU.td

Show First 20 Lines • Show All 405 Lines • ▼ Show 20 Lines
>; >;
def FeatureGCN : SubtargetFeature<"gcn", def FeatureGCN : SubtargetFeature<"gcn",
"IsGCN", "IsGCN",
"true", "true",
"GCN or newer GPU" "GCN or newer GPU"
>; >;
class AMDGPUSubtargetFeatureGeneration <string Value, class GCNSubtargetFeatureGeneration <string Value,
list<SubtargetFeature> Implies> : list<SubtargetFeature> Implies> :
SubtargetFeatureGeneration <Value, "AMDGPUSubtarget", Implies>; SubtargetFeatureGeneration <Value, "GCNSubtarget", Implies>;
def FeatureSouthernIslands : AMDGPUSubtargetFeatureGeneration<"SOUTHERN_ISLANDS", def FeatureSouthernIslands : GCNSubtargetFeatureGeneration<"SOUTHERN_ISLANDS",
[FeatureFP64, FeatureLocalMemorySize32768, FeatureMIMG_R128, [FeatureFP64, FeatureLocalMemorySize32768, FeatureMIMG_R128,
FeatureWavefrontSize64, FeatureGCN, FeatureWavefrontSize64, FeatureGCN,
FeatureLDSBankCount32, FeatureMovrel] FeatureLDSBankCount32, FeatureMovrel]
>; >;
def FeatureSeaIslands : AMDGPUSubtargetFeatureGeneration<"SEA_ISLANDS", def FeatureSeaIslands : GCNSubtargetFeatureGeneration<"SEA_ISLANDS",
[FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128, [FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128,
FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace, FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace,
FeatureCIInsts, FeatureMovrel] FeatureCIInsts, FeatureMovrel]
>; >;
def FeatureVolcanicIslands : AMDGPUSubtargetFeatureGeneration<"VOLCANIC_ISLANDS", def FeatureVolcanicIslands : GCNSubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
[FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128, [FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128,
FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN, FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts, FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts,
FeatureSMemRealTime, FeatureVGPRIndexMode, FeatureMovrel, FeatureSMemRealTime, FeatureVGPRIndexMode, FeatureMovrel,
FeatureScalarStores, FeatureInv2PiInlineImm, FeatureScalarStores, FeatureInv2PiInlineImm,
FeatureSDWA, FeatureSDWAOutModsVOPC, FeatureSDWAMac, FeatureDPP, FeatureSDWA, FeatureSDWAOutModsVOPC, FeatureSDWAMac, FeatureDPP,
FeatureIntClamp FeatureIntClamp
] ]
>; >;
def FeatureGFX9 : AMDGPUSubtargetFeatureGeneration<"GFX9", def FeatureGFX9 : GCNSubtargetFeatureGeneration<"GFX9",
[FeatureFP64, FeatureLocalMemorySize65536, [FeatureFP64, FeatureLocalMemorySize65536,
FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN, FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts, FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts,
FeatureSMemRealTime, FeatureScalarStores, FeatureInv2PiInlineImm, FeatureSMemRealTime, FeatureScalarStores, FeatureInv2PiInlineImm,
FeatureApertureRegs, FeatureGFX9Insts, FeatureVOP3P, FeatureVGPRIndexMode, FeatureApertureRegs, FeatureGFX9Insts, FeatureVOP3P, FeatureVGPRIndexMode,
FeatureFastFMAF32, FeatureDPP, FeatureIntClamp, FeatureFastFMAF32, FeatureDPP, FeatureIntClamp,
FeatureSDWA, FeatureSDWAOmod, FeatureSDWAScalar, FeatureSDWASdst, FeatureSDWA, FeatureSDWAOmod, FeatureSDWAScalar, FeatureSDWASdst,
FeatureFlatInstOffsets, FeatureFlatGlobalInsts, FeatureFlatScratchInsts, FeatureFlatInstOffsets, FeatureFlatGlobalInsts, FeatureFlatScratchInsts,
▲ Show 20 Lines • Show All 187 Lines • ▼ Show 20 Lines
def ALU_NULL : FuncUnit; def ALU_NULL : FuncUnit;
def NullALU : InstrItinClass; def NullALU : InstrItinClass;
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// Predicate helper class // Predicate helper class
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
def isSICI : Predicate< def isSICI : Predicate<
"Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS ||" "Subtarget->getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS ||"
"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS" "Subtarget->getGeneration() == GCNSubtarget::SEA_ISLANDS"
>, AssemblerPredicate<"!FeatureGCN3Encoding">; >, AssemblerPredicate<"!FeatureGCN3Encoding">;
def isVI : Predicate < def isVI : Predicate <
"Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS">, "Subtarget->getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS">,
AssemblerPredicate<"FeatureGCN3Encoding">; AssemblerPredicate<"FeatureGCN3Encoding">;
def isGFX9 : Predicate < def isGFX9 : Predicate <
"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">, "Subtarget->getGeneration() >= GCNSubtarget::GFX9">,
AssemblerPredicate<"FeatureGFX9Insts">; AssemblerPredicate<"FeatureGFX9Insts">;
// TODO: Either the name to be changed or we simply use IsCI! // TODO: Either the name to be changed or we simply use IsCI!
def isCIVI : Predicate < def isCIVI : Predicate <
"Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS">, "Subtarget->getGeneration() >= GCNSubtarget::SEA_ISLANDS">,
AssemblerPredicate<"FeatureCIInsts">; AssemblerPredicate<"FeatureCIInsts">;
def HasFlatAddressSpace : Predicate<"Subtarget->hasFlatAddressSpace()">, def HasFlatAddressSpace : Predicate<"Subtarget->hasFlatAddressSpace()">,
AssemblerPredicate<"FeatureFlatAddressSpace">; AssemblerPredicate<"FeatureFlatAddressSpace">;
def HasFlatGlobalInsts : Predicate<"Subtarget->hasFlatGlobalInsts()">, def HasFlatGlobalInsts : Predicate<"Subtarget->hasFlatGlobalInsts()">,
AssemblerPredicate<"FeatureFlatGlobalInsts">; AssemblerPredicate<"FeatureFlatGlobalInsts">;
def HasFlatScratchInsts : Predicate<"Subtarget->hasFlatScratchInsts()">, def HasFlatScratchInsts : Predicate<"Subtarget->hasFlatScratchInsts()">,
AssemblerPredicate<"FeatureFlatScratchInsts">; AssemblerPredicate<"FeatureFlatScratchInsts">;
def HasD16LoadStore : Predicate<"Subtarget->hasD16LoadStore()">, def HasD16LoadStore : Predicate<"Subtarget->hasD16LoadStore()">,
AssemblerPredicate<"FeatureGFX9Insts">; AssemblerPredicate<"FeatureGFX9Insts">;
def HasUnpackedD16VMem : Predicate<"Subtarget->hasUnpackedD16VMem()">, def HasUnpackedD16VMem : Predicate<"Subtarget->hasUnpackedD16VMem()">,
AssemblerPredicate<"FeatureUnpackedD16VMem">; AssemblerPredicate<"FeatureUnpackedD16VMem">;
def HasPackedD16VMem : Predicate<"!Subtarget->hasUnpackedD16VMem()">, def HasPackedD16VMem : Predicate<"!Subtarget->hasUnpackedD16VMem()">,
AssemblerPredicate<"!FeatureUnpackedD16VMem">; AssemblerPredicate<"!FeatureUnpackedD16VMem">;
def D16PreservesUnusedBits : Predicate<"Subtarget->d16PreservesUnusedBits()">, def D16PreservesUnusedBits : Predicate<"Subtarget->d16PreservesUnusedBits()">,
AssemblerPredicate<"FeatureD16PreservesUnusedBits">; AssemblerPredicate<"FeatureD16PreservesUnusedBits">;
def LDSRequiresM0Init : Predicate<"Subtarget->ldsRequiresM0Init()">; def LDSRequiresM0Init : Predicate<"Subtarget->ldsRequiresM0Init()">;
def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">; def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">;
def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">, def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= GCNSubtarget::GFX9">,
AssemblerPredicate<"FeatureGFX9Insts">; AssemblerPredicate<"FeatureGFX9Insts">;
def HasAddNoCarryInsts : Predicate<"Subtarget->hasAddNoCarryInsts()">, def HasAddNoCarryInsts : Predicate<"Subtarget->hasAddNoCarryInsts()">,
AssemblerPredicate<"FeatureAddNoCarryInsts">; AssemblerPredicate<"FeatureAddNoCarryInsts">;
def NotHasAddNoCarryInsts : Predicate<"!Subtarget->hasAddNoCarryInsts()">, def NotHasAddNoCarryInsts : Predicate<"!Subtarget->hasAddNoCarryInsts()">,
AssemblerPredicate<"!FeatureAddNoCarryInsts">; AssemblerPredicate<"!FeatureAddNoCarryInsts">;
▲ Show 20 Lines • Show All 55 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp

Show First 20 Lines • Show All 213 Lines • ▼ Show 20 Linesstatic void copyFeaturesToFunction(Function &Parent, const Function &Callee,
if (handleAttr(Parent, Callee, "amdgpu-queue-ptr")) if (handleAttr(Parent, Callee, "amdgpu-queue-ptr"))
NeedQueuePtr = true; NeedQueuePtr = true;
for (StringRef AttrName : AttrNames) for (StringRef AttrName : AttrNames)
handleAttr(Parent, Callee, AttrName); handleAttr(Parent, Callee, AttrName);
} }
bool AMDGPUAnnotateKernelFeatures::addFeatureAttributes(Function &F) { bool AMDGPUAnnotateKernelFeatures::addFeatureAttributes(Function &F) {
const AMDGPUSubtarget &ST = TM->getSubtarget<AMDGPUSubtarget>(F); const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(F);
bool HasFlat = ST.hasFlatAddressSpace(); bool HasFlat = ST.hasFlatAddressSpace();
bool HasApertureRegs = ST.hasApertureRegs(); bool HasApertureRegs = ST.hasApertureRegs();
SmallPtrSet<const Constant *, 8> ConstantExprVisited; SmallPtrSet<const Constant *, 8> ConstantExprVisited;
bool Changed = false; bool Changed = false;
bool NeedQueuePtr = false; bool NeedQueuePtr = false;
bool HaveCall = false; bool HaveCall = false;
bool IsFunc = !AMDGPU::isEntryFunctionCC(F.getCallingConv()); bool IsFunc = !AMDGPU::isEntryFunctionCC(F.getCallingConv());
▲ Show 20 Lines • Show All 98 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h

Show All 12 Lines
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/IR/Function.h" #include "llvm/IR/Function.h"
#include "llvm/Pass.h" #include "llvm/Pass.h"
namespace llvm { namespace llvm {
class Function; class Function;
class raw_ostream; class raw_ostream;
class SISubtarget; class GCNSubtarget;
class TargetMachine; class TargetMachine;
class TargetRegisterClass; class TargetRegisterClass;
class TargetRegisterInfo; class TargetRegisterInfo;
struct ArgDescriptor { struct ArgDescriptor {
private: private:
friend struct AMDGPUFunctionArgInfo; friend struct AMDGPUFunctionArgInfo;
friend class AMDGPUArgumentUsageInfo; friend class AMDGPUArgumentUsageInfo;
▲ Show 20 Lines • Show All 145 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUAsmPrinter.h

Show All 27 Lines
#include <string> #include <string>
#include <vector> #include <vector>
namespace llvm { namespace llvm {
class AMDGPUMachineFunction; class AMDGPUMachineFunction;
class AMDGPUTargetStreamer; class AMDGPUTargetStreamer;
class MCOperand; class MCOperand;
class SISubtarget; class GCNSubtarget;
class AMDGPUAsmPrinter final : public AsmPrinter { class AMDGPUAsmPrinter final : public AsmPrinter {
private: private:
// Track resource usage for callee functions. // Track resource usage for callee functions.
struct SIFunctionResourceInfo { struct SIFunctionResourceInfo {
// Track the number of explicitly used VGPRs. Special registers reserved at // Track the number of explicitly used VGPRs. Special registers reserved at
// the end are tracked separately. // the end are tracked separately.
int32_t NumVGPR = 0; int32_t NumVGPR = 0;
int32_t NumExplicitSGPR = 0; int32_t NumExplicitSGPR = 0;
uint64_t PrivateSegmentSize = 0; uint64_t PrivateSegmentSize = 0;
bool UsesVCC = false; bool UsesVCC = false;
bool UsesFlatScratch = false; bool UsesFlatScratch = false;
bool HasDynamicallySizedStack = false; bool HasDynamicallySizedStack = false;
bool HasRecursion = false; bool HasRecursion = false;
int32_t getTotalNumSGPRs(const SISubtarget &ST) const; int32_t getTotalNumSGPRs(const GCNSubtarget &ST) const;
}; };
// Track resource usage for kernels / entry functions. // Track resource usage for kernels / entry functions.
struct SIProgramInfo { struct SIProgramInfo {
// Fields set in PGM_RSRC1 pm4 packet. // Fields set in PGM_RSRC1 pm4 packet.
uint32_t VGPRBlocks = 0; uint32_t VGPRBlocks = 0;
uint32_t SGPRBlocks = 0; uint32_t SGPRBlocks = 0;
uint32_t Priority = 0; uint32_t Priority = 0;
▲ Show 20 Lines • Show All 151 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

Show First 20 Lines • Show All 61 Lines • ▼ Show 20 Lines
// FIXME: It seems some instructions do not support single precision denormals // FIXME: It seems some instructions do not support single precision denormals
// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32, // regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
// and sin_f32, cos_f32 on most parts). // and sin_f32, cos_f32 on most parts).
// We want to use these instructions, and using fp32 denormals also causes // We want to use these instructions, and using fp32 denormals also causes
// instructions to run at the double precision rate for the device so it's // instructions to run at the double precision rate for the device so it's
// probably best to just report no single precision denormals. // probably best to just report no single precision denormals.
static uint32_t getFPMode(const MachineFunction &F) { static uint32_t getFPMode(const MachineFunction &F) {
const SISubtarget& ST = F.getSubtarget<SISubtarget>(); const GCNSubtarget& ST = F.getSubtarget<GCNSubtarget>();
// TODO: Is there any real use for the flush in only / flush out only modes? // TODO: Is there any real use for the flush in only / flush out only modes?
uint32_t FP32Denormals = uint32_t FP32Denormals =
ST.hasFP32Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; ST.hasFP32Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT;
uint32_t FP64Denormals = uint32_t FP64Denormals =
ST.hasFP64Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; ST.hasFP64Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT;
▲ Show 20 Lines • Show All 113 Lines • ▼ Show 20 Lines
void AMDGPUAsmPrinter::EmitFunctionBodyStart() { void AMDGPUAsmPrinter::EmitFunctionBodyStart() {
const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
if (!MFI.isEntryFunction()) if (!MFI.isEntryFunction())
return; return;
if (IsaInfo::hasCodeObjectV3(getSTI()) && if (IsaInfo::hasCodeObjectV3(getSTI()) &&
TM.getTargetTriple().getOS() == Triple::AMDHSA) TM.getTargetTriple().getOS() == Triple::AMDHSA)
return; return;
const AMDGPUSubtarget &STM = MF->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
amd_kernel_code_t KernelCode; amd_kernel_code_t KernelCode;
if (STM.isAmdCodeObjectV2(MF->getFunction())) { if (STM.isAmdCodeObjectV2(MF->getFunction())) {
getAmdKernelCode(KernelCode, CurrentProgramInfo, *MF); getAmdKernelCode(KernelCode, CurrentProgramInfo, *MF);
getTargetStreamer()->EmitAMDKernelCodeT(KernelCode); getTargetStreamer()->EmitAMDKernelCodeT(KernelCode);
} }
if (TM.getTargetTriple().getOS() != Triple::AMDHSA) if (TM.getTargetTriple().getOS() != Triple::AMDHSA)
return; return;
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines
void AMDGPUAsmPrinter::EmitFunctionEntryLabel() { void AMDGPUAsmPrinter::EmitFunctionEntryLabel() {
if (IsaInfo::hasCodeObjectV3(getSTI()) && if (IsaInfo::hasCodeObjectV3(getSTI()) &&
TM.getTargetTriple().getOS() == Triple::AMDHSA) { TM.getTargetTriple().getOS() == Triple::AMDHSA) {
AsmPrinter::EmitFunctionEntryLabel(); AsmPrinter::EmitFunctionEntryLabel();
return; return;
} }
const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
const AMDGPUSubtarget &STM = MF->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
if (MFI->isEntryFunction() && STM.isAmdCodeObjectV2(MF->getFunction())) { if (MFI->isEntryFunction() && STM.isAmdCodeObjectV2(MF->getFunction())) {
SmallString<128> SymbolName; SmallString<128> SymbolName;
getNameWithPrefix(SymbolName, &MF->getFunction()), getNameWithPrefix(SymbolName, &MF->getFunction()),
getTargetStreamer()->EmitAMDGPUSymbolType( getTargetStreamer()->EmitAMDGPUSymbolType(
SymbolName, ELF::STT_AMDGPU_HSA_KERNEL); SymbolName, ELF::STT_AMDGPU_HSA_KERNEL);
} }
const AMDGPUSubtarget &STI = MF->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();
if (STI.dumpCode()) { if (STI.dumpCode()) {
// Disassemble function name label to text. // Disassemble function name label to text.
DisasmLines.push_back(MF->getName().str() + ":"); DisasmLines.push_back(MF->getName().str() + ":");
DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLines.back().size()); DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLines.back().size());
HexLines.push_back(""); HexLines.push_back("");
} }
AsmPrinter::EmitFunctionEntryLabel(); AsmPrinter::EmitFunctionEntryLabel();
} }
void AMDGPUAsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) const { void AMDGPUAsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) const {
const AMDGPUSubtarget &STI = MBB.getParent()->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STI = MBB.getParent()->getSubtarget<GCNSubtarget>();
if (STI.dumpCode() && !isBlockOnlyReachableByFallthrough(&MBB)) { if (STI.dumpCode() && !isBlockOnlyReachableByFallthrough(&MBB)) {
// Write a line for the basic block label if it is not only fallthrough. // Write a line for the basic block label if it is not only fallthrough.
DisasmLines.push_back( DisasmLines.push_back(
(Twine("BB") + Twine(getFunctionNumber()) (Twine("BB") + Twine(getFunctionNumber())
+ "_" + Twine(MBB.getNumber()) + ":").str()); + "_" + Twine(MBB.getNumber()) + ":").str());
DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLines.back().size()); DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLines.back().size());
HexLines.push_back(""); HexLines.push_back("");
} }
▲ Show 20 Lines • Show All 108 Lines • ▼ Show 20 Linesbool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>(); const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
// The starting address of all shader programs must be 256 bytes aligned. // The starting address of all shader programs must be 256 bytes aligned.
// Regular functions just need the basic required instruction alignment. // Regular functions just need the basic required instruction alignment.
MF.setAlignment(MFI->isEntryFunction() ? 8 : 2); MF.setAlignment(MFI->isEntryFunction() ? 8 : 2);
SetupMachineFunction(MF); SetupMachineFunction(MF);
const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
MCContext &Context = getObjFileLowering().getContext(); MCContext &Context = getObjFileLowering().getContext();
// FIXME: This should be an explicit check for Mesa. // FIXME: This should be an explicit check for Mesa.
if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) { if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
MCSectionELF *ConfigSection = MCSectionELF *ConfigSection =
Context.getELFSection(".AMDGPU.config", ELF::SHT_PROGBITS, 0); Context.getELFSection(".AMDGPU.config", ELF::SHT_PROGBITS, 0);
OutStreamer->SwitchSection(ConfigSection); OutStreamer->SwitchSection(ConfigSection);
} }
Show All 24 Lines MCSectionELF *CommentSection =
Context.getELFSection(".AMDGPU.csdata", ELF::SHT_PROGBITS, 0); Context.getELFSection(".AMDGPU.csdata", ELF::SHT_PROGBITS, 0);
OutStreamer->SwitchSection(CommentSection); OutStreamer->SwitchSection(CommentSection);
if (!MFI->isEntryFunction()) { if (!MFI->isEntryFunction()) {
OutStreamer->emitRawComment(" Function info:", false); OutStreamer->emitRawComment(" Function info:", false);
SIFunctionResourceInfo &Info = CallGraphResourceInfo[&MF.getFunction()]; SIFunctionResourceInfo &Info = CallGraphResourceInfo[&MF.getFunction()];
emitCommonFunctionComments( emitCommonFunctionComments(
Info.NumVGPR, Info.NumVGPR,
Info.getTotalNumSGPRs(MF.getSubtarget<SISubtarget>()), Info.getTotalNumSGPRs(MF.getSubtarget<GCNSubtarget>()),
Info.PrivateSegmentSize, Info.PrivateSegmentSize,
getFunctionCodeSize(MF), MFI); getFunctionCodeSize(MF), MFI);
return false; return false;
} }
OutStreamer->emitRawComment(" Kernel info:", false); OutStreamer->emitRawComment(" Kernel info:", false);
emitCommonFunctionComments(CurrentProgramInfo.NumVGPR, emitCommonFunctionComments(CurrentProgramInfo.NumVGPR,
CurrentProgramInfo.NumSGPR, CurrentProgramInfo.NumSGPR,
Show All 18 Lines OutStreamer->emitRawComment(
Twine(CurrentProgramInfo.NumSGPRsForWavesPerEU), false); Twine(CurrentProgramInfo.NumSGPRsForWavesPerEU), false);
OutStreamer->emitRawComment( OutStreamer->emitRawComment(
" NumVGPRsForWavesPerEU: " + " NumVGPRsForWavesPerEU: " +
Twine(CurrentProgramInfo.NumVGPRsForWavesPerEU), false); Twine(CurrentProgramInfo.NumVGPRsForWavesPerEU), false);
OutStreamer->emitRawComment( OutStreamer->emitRawComment(
" WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), false); " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), false);
if (MF.getSubtarget<SISubtarget>().debuggerEmitPrologue()) { if (MF.getSubtarget<GCNSubtarget>().debuggerEmitPrologue()) {
OutStreamer->emitRawComment( OutStreamer->emitRawComment(
" DebuggerWavefrontPrivateSegmentOffsetSGPR: s" + " DebuggerWavefrontPrivateSegmentOffsetSGPR: s" +
Twine(CurrentProgramInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR), false); Twine(CurrentProgramInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR), false);
OutStreamer->emitRawComment( OutStreamer->emitRawComment(
" DebuggerPrivateSegmentBufferSGPR: s" + " DebuggerPrivateSegmentBufferSGPR: s" +
Twine(CurrentProgramInfo.DebuggerPrivateSegmentBufferSGPR), false); Twine(CurrentProgramInfo.DebuggerPrivateSegmentBufferSGPR), false);
} }
Show All 34 Lines for (size_t i = 0; i < DisasmLines.size(); ++i) {
OutStreamer->EmitBytes(StringRef(Comment)); OutStreamer->EmitBytes(StringRef(Comment));
} }
} }
return false; return false;
} }
uint64_t AMDGPUAsmPrinter::getFunctionCodeSize(const MachineFunction &MF) const { uint64_t AMDGPUAsmPrinter::getFunctionCodeSize(const MachineFunction &MF) const {
const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = STM.getInstrInfo(); const SIInstrInfo *TII = STM.getInstrInfo();
uint64_t CodeSize = 0; uint64_t CodeSize = 0;
for (const MachineBasicBlock &MBB : MF) { for (const MachineBasicBlock &MBB : MF) {
for (const MachineInstr &MI : MBB) { for (const MachineInstr &MI : MBB) {
// TODO: CodeSize should account for multiple functions. // TODO: CodeSize should account for multiple functions.
Show All 15 Lines for (const MachineOperand &UseOp : MRI.reg_operands(Reg)) {
if (!UseOp.isImplicit() || !TII.isFLAT(*UseOp.getParent())) if (!UseOp.isImplicit() || !TII.isFLAT(*UseOp.getParent()))
return true; return true;
} }
return false; return false;
} }
int32_t AMDGPUAsmPrinter::SIFunctionResourceInfo::getTotalNumSGPRs( int32_t AMDGPUAsmPrinter::SIFunctionResourceInfo::getTotalNumSGPRs(
const SISubtarget &ST) const { const GCNSubtarget &ST) const {
return NumExplicitSGPR + IsaInfo::getNumExtraSGPRs(ST.getFeatureBits(), return NumExplicitSGPR + IsaInfo::getNumExtraSGPRs(ST.getFeatureBits(),
UsesVCC, UsesFlatScratch); UsesVCC, UsesFlatScratch);
} }
AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage( AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
const MachineFunction &MF) const { const MachineFunction &MF) const {
SIFunctionResourceInfo Info; SIFunctionResourceInfo Info;
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const MachineFrameInfo &FrameInfo = MF.getFrameInfo(); const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
const MachineRegisterInfo &MRI = MF.getRegInfo(); const MachineRegisterInfo &MRI = MF.getRegInfo();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo &TRI = TII->getRegisterInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo();
Info.UsesFlatScratch = MRI.isPhysRegUsed(AMDGPU::FLAT_SCR_LO) || Info.UsesFlatScratch = MRI.isPhysRegUsed(AMDGPU::FLAT_SCR_LO) ||
MRI.isPhysRegUsed(AMDGPU::FLAT_SCR_HI); MRI.isPhysRegUsed(AMDGPU::FLAT_SCR_HI);
▲ Show 20 Lines • Show All 227 Lines • ▼ Show 20 Linesvoid AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
ProgInfo.DynamicCallStack = Info.HasDynamicallySizedStack || Info.HasRecursion; ProgInfo.DynamicCallStack = Info.HasDynamicallySizedStack || Info.HasRecursion;
if (!isUInt<32>(ProgInfo.ScratchSize)) { if (!isUInt<32>(ProgInfo.ScratchSize)) {
DiagnosticInfoStackSize DiagStackSize(MF.getFunction(), DiagnosticInfoStackSize DiagStackSize(MF.getFunction(),
ProgInfo.ScratchSize, DS_Error); ProgInfo.ScratchSize, DS_Error);
MF.getFunction().getContext().diagnose(DiagStackSize); MF.getFunction().getContext().diagnose(DiagStackSize);
} }
const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
const SIInstrInfo *TII = STM.getInstrInfo(); const SIInstrInfo *TII = STM.getInstrInfo();
const SIRegisterInfo *RI = &TII->getRegisterInfo(); const SIRegisterInfo *RI = &TII->getRegisterInfo();
// TODO(scott.linder): The calculations related to SGPR/VGPR blocks are // TODO(scott.linder): The calculations related to SGPR/VGPR blocks are
// duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
// unified. // unified.
unsigned ExtraSGPRs = IsaInfo::getNumExtraSGPRs( unsigned ExtraSGPRs = IsaInfo::getNumExtraSGPRs(
STM.getFeatureBits(), ProgInfo.VCCUsed, ProgInfo.FlatUsed); STM.getFeatureBits(), ProgInfo.VCCUsed, ProgInfo.FlatUsed);
// Check the addressable register limit before we add ExtraSGPRs. // Check the addressable register limit before we add ExtraSGPRs.
if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && if (STM.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS &&
!STM.hasSGPRInitBug()) { !STM.hasSGPRInitBug()) {
unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs(); unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
if (ProgInfo.NumSGPR > MaxAddressableNumSGPRs) { if (ProgInfo.NumSGPR > MaxAddressableNumSGPRs) {
// This can happen due to a compiler bug or when using inline asm. // This can happen due to a compiler bug or when using inline asm.
LLVMContext &Ctx = MF.getFunction().getContext(); LLVMContext &Ctx = MF.getFunction().getContext();
DiagnosticInfoResourceLimit Diag(MF.getFunction(), DiagnosticInfoResourceLimit Diag(MF.getFunction(),
"addressable scalar registers", "addressable scalar registers",
ProgInfo.NumSGPR, DS_Error, ProgInfo.NumSGPR, DS_Error,
Show All 22 Linesvoid AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
// Adjust number of registers used to meet default/requested minimum/maximum // Adjust number of registers used to meet default/requested minimum/maximum
// number of waves per execution unit request. // number of waves per execution unit request.
ProgInfo.NumSGPRsForWavesPerEU = std::max( ProgInfo.NumSGPRsForWavesPerEU = std::max(
std::max(ProgInfo.NumSGPR, 1u), STM.getMinNumSGPRs(MFI->getMaxWavesPerEU())); std::max(ProgInfo.NumSGPR, 1u), STM.getMinNumSGPRs(MFI->getMaxWavesPerEU()));
ProgInfo.NumVGPRsForWavesPerEU = std::max( ProgInfo.NumVGPRsForWavesPerEU = std::max(
std::max(ProgInfo.NumVGPR, 1u), STM.getMinNumVGPRs(MFI->getMaxWavesPerEU())); std::max(ProgInfo.NumVGPR, 1u), STM.getMinNumVGPRs(MFI->getMaxWavesPerEU()));
if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS || if (STM.getGeneration() <= GCNSubtarget::SEA_ISLANDS ||
STM.hasSGPRInitBug()) { STM.hasSGPRInitBug()) {
unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs(); unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
if (ProgInfo.NumSGPR > MaxAddressableNumSGPRs) { if (ProgInfo.NumSGPR > MaxAddressableNumSGPRs) {
// This can happen due to a compiler bug or when using inline asm to use // This can happen due to a compiler bug or when using inline asm to use
// the registers which are usually reserved for vcc etc. // the registers which are usually reserved for vcc etc.
LLVMContext &Ctx = MF.getFunction().getContext(); LLVMContext &Ctx = MF.getFunction().getContext();
DiagnosticInfoResourceLimit Diag(MF.getFunction(), DiagnosticInfoResourceLimit Diag(MF.getFunction(),
"scalar registers", "scalar registers",
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Linesvoid AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
ProgInfo.FloatMode = getFPMode(MF); ProgInfo.FloatMode = getFPMode(MF);
ProgInfo.IEEEMode = STM.enableIEEEBit(MF); ProgInfo.IEEEMode = STM.enableIEEEBit(MF);
// Make clamp modifier on NaN input returns 0. // Make clamp modifier on NaN input returns 0.
ProgInfo.DX10Clamp = STM.enableDX10Clamp(); ProgInfo.DX10Clamp = STM.enableDX10Clamp();
unsigned LDSAlignShift; unsigned LDSAlignShift;
if (STM.getGeneration() < SISubtarget::SEA_ISLANDS) { if (STM.getGeneration() < GCNSubtarget::SEA_ISLANDS) {
// LDS is allocated in 64 dword blocks. // LDS is allocated in 64 dword blocks.
LDSAlignShift = 8; LDSAlignShift = 8;
} else { } else {
// LDS is allocated in 128 dword blocks. // LDS is allocated in 128 dword blocks.
LDSAlignShift = 9; LDSAlignShift = 9;
} }
unsigned LDSSpillSize = unsigned LDSSpillSize =
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Linesstatic unsigned getRsrcReg(CallingConv::ID CallConv) {
case CallingConv::AMDGPU_GS: return R_00B228_SPI_SHADER_PGM_RSRC1_GS; case CallingConv::AMDGPU_GS: return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
case CallingConv::AMDGPU_VS: return R_00B128_SPI_SHADER_PGM_RSRC1_VS; case CallingConv::AMDGPU_VS: return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
case CallingConv::AMDGPU_PS: return R_00B028_SPI_SHADER_PGM_RSRC1_PS; case CallingConv::AMDGPU_PS: return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
} }
} }
void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF, void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
const SIProgramInfo &CurrentProgramInfo) { const SIProgramInfo &CurrentProgramInfo) {
const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
unsigned RsrcReg = getRsrcReg(MF.getFunction().getCallingConv()); unsigned RsrcReg = getRsrcReg(MF.getFunction().getCallingConv());
if (AMDGPU::isCompute(MF.getFunction().getCallingConv())) { if (AMDGPU::isCompute(MF.getFunction().getCallingConv())) {
OutStreamer->EmitIntValue(R_00B848_COMPUTE_PGM_RSRC1, 4); OutStreamer->EmitIntValue(R_00B848_COMPUTE_PGM_RSRC1, 4);
OutStreamer->EmitIntValue(CurrentProgramInfo.ComputePGMRSrc1, 4); OutStreamer->EmitIntValue(CurrentProgramInfo.ComputePGMRSrc1, 4);
▲ Show 20 Lines • Show All 112 Lines • ▼ Show 20 Lines default:
llvm_unreachable("invalid private_element_size"); llvm_unreachable("invalid private_element_size");
} }
} }
void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out, void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out,
const SIProgramInfo &CurrentProgramInfo, const SIProgramInfo &CurrentProgramInfo,
const MachineFunction &MF) const { const MachineFunction &MF) const {
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
AMDGPU::initDefaultAMDKernelCodeT(Out, STM.getFeatureBits()); AMDGPU::initDefaultAMDKernelCodeT(Out, STM.getFeatureBits());
Out.compute_pgm_resource_registers = Out.compute_pgm_resource_registers =
CurrentProgramInfo.ComputePGMRSrc1 | CurrentProgramInfo.ComputePGMRSrc1 |
(CurrentProgramInfo.ComputePGMRSrc2 << 32); (CurrentProgramInfo.ComputePGMRSrc2 << 32);
Out.code_properties = AMD_CODE_PROPERTY_IS_PTR64; Out.code_properties = AMD_CODE_PROPERTY_IS_PTR64;
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines if (STM.debuggerEmitPrologue()) {
Out.debug_private_segment_buffer_sgpr = Out.debug_private_segment_buffer_sgpr =
CurrentProgramInfo.DebuggerPrivateSegmentBufferSGPR; CurrentProgramInfo.DebuggerPrivateSegmentBufferSGPR;
} }
} }
AMDGPU::HSAMD::Kernel::CodeProps::Metadata AMDGPUAsmPrinter::getHSACodeProps( AMDGPU::HSAMD::Kernel::CodeProps::Metadata AMDGPUAsmPrinter::getHSACodeProps(
const MachineFunction &MF, const MachineFunction &MF,
const SIProgramInfo &ProgramInfo) const { const SIProgramInfo &ProgramInfo) const {
const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
HSAMD::Kernel::CodeProps::Metadata HSACodeProps; HSAMD::Kernel::CodeProps::Metadata HSACodeProps;
const Function &F = MF.getFunction(); const Function &F = MF.getFunction();
// Avoid asserting on erroneous cases. // Avoid asserting on erroneous cases.
if (F.getCallingConv() != CallingConv::AMDGPU_KERNEL) if (F.getCallingConv() != CallingConv::AMDGPU_KERNEL)
return HSACodeProps; return HSACodeProps;
Show All 12 LinesAMDGPU::HSAMD::Kernel::CodeProps::Metadata AMDGPUAsmPrinter::getHSACodeProps(
HSACodeProps.mNumSpilledVGPRs = MFI.getNumSpilledVGPRs(); HSACodeProps.mNumSpilledVGPRs = MFI.getNumSpilledVGPRs();
return HSACodeProps; return HSACodeProps;
} }
AMDGPU::HSAMD::Kernel::DebugProps::Metadata AMDGPUAsmPrinter::getHSADebugProps( AMDGPU::HSAMD::Kernel::DebugProps::Metadata AMDGPUAsmPrinter::getHSADebugProps(
const MachineFunction &MF, const MachineFunction &MF,
const SIProgramInfo &ProgramInfo) const { const SIProgramInfo &ProgramInfo) const {
const SISubtarget &STM = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
HSAMD::Kernel::DebugProps::Metadata HSADebugProps; HSAMD::Kernel::DebugProps::Metadata HSADebugProps;
if (!STM.debuggerSupported()) if (!STM.debuggerSupported())
return HSADebugProps; return HSADebugProps;
HSADebugProps.mDebuggerABIVersion.push_back(1); HSADebugProps.mDebuggerABIVersion.push_back(1);
HSADebugProps.mDebuggerABIVersion.push_back(0); HSADebugProps.mDebuggerABIVersion.push_back(0);
Show All 39 Lines

lib/Target/AMDGPU/AMDGPUCallLowering.cpp

Show First 20 Lines • Show All 89 Lines • ▼ Show 20 Linesbool AMDGPUCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
const Function &F, const Function &F,
ArrayRef<unsigned> VRegs) const { ArrayRef<unsigned> VRegs) const {
// AMDGPU_GS and AMDGP_HS are not supported yet. // AMDGPU_GS and AMDGP_HS are not supported yet.
if (F.getCallingConv() == CallingConv::AMDGPU_GS || if (F.getCallingConv() == CallingConv::AMDGPU_GS ||
F.getCallingConv() == CallingConv::AMDGPU_HS) F.getCallingConv() == CallingConv::AMDGPU_HS)
return false; return false;
MachineFunction &MF = MIRBuilder.getMF(); MachineFunction &MF = MIRBuilder.getMF();
const SISubtarget *Subtarget = static_cast<const SISubtarget *>(&MF.getSubtarget()); const GCNSubtarget *Subtarget = static_cast<const GCNSubtarget *>(&MF.getSubtarget());
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
const SIRegisterInfo *TRI = MF.getSubtarget<SISubtarget>().getRegisterInfo(); const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
const DataLayout &DL = F.getParent()->getDataLayout(); const DataLayout &DL = F.getParent()->getDataLayout();
SmallVector<CCValAssign, 16> ArgLocs; SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext()); CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
// FIXME: How should these inputs interact with inreg / custom SGPR inputs? // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
if (Info->hasPrivateSegmentBuffer()) { if (Info->hasPrivateSegmentBuffer()) {
unsigned PrivateSegmentBufferReg = Info->addPrivateSegmentBuffer(*TRI); unsigned PrivateSegmentBufferReg = Info->addPrivateSegmentBuffer(*TRI);
▲ Show 20 Lines • Show All 119 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUCallingConv.td

Show First 20 Lines • Show All 131 Lines • ▼ Show 20 Lines CCIfType<[i32, f32, i16, f16, v2i16, v2f16], CCAssignToReg<[
VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7, VGPR0, VGPR1, VGPR2, VGPR3, VGPR4, VGPR5, VGPR6, VGPR7,
VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15, VGPR8, VGPR9, VGPR10, VGPR11, VGPR12, VGPR13, VGPR14, VGPR15,
VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23, VGPR16, VGPR17, VGPR18, VGPR19, VGPR20, VGPR21, VGPR22, VGPR23,
VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31]>>, VGPR24, VGPR25, VGPR26, VGPR27, VGPR28, VGPR29, VGPR30, VGPR31]>>,
CCIfType<[i64, f64, v2i32, v2f32, v4i32, v4f32, v8i32, v8f32, v16i32, v16f32, v2i64, v2f64, v4i16, v4f16], CCCustom<"allocateVGPRTuple">> CCIfType<[i64, f64, v2i32, v2f32, v4i32, v4f32, v8i32, v8f32, v16i32, v16f32, v2i64, v2f64, v4i16, v4f16], CCCustom<"allocateVGPRTuple">>
]>; ]>;
def CC_AMDGPU : CallingConv<[ def CC_AMDGPU : CallingConv<[
CCIf<"static_cast<const AMDGPUSubtarget&>" CCIf<"static_cast<const GCNSubtarget&>"
"(State.getMachineFunction().getSubtarget()).getGeneration() >=" "(State.getMachineFunction().getSubtarget()).getGeneration() >="
"AMDGPUSubtarget::SOUTHERN_ISLANDS && " "GCNSubtarget::SOUTHERN_ISLANDS && "
"!AMDGPU::isShader(State.getCallingConv())", "!AMDGPU::isShader(State.getCallingConv())",
CCDelegateTo<CC_AMDGPU_Kernel>>, CCDelegateTo<CC_AMDGPU_Kernel>>,
CCIf<"static_cast<const AMDGPUSubtarget&>" CCIf<"static_cast<const GCNSubtarget&>"
"(State.getMachineFunction().getSubtarget()).getGeneration() < " "(State.getMachineFunction().getSubtarget()).getGeneration() < "
"AMDGPUSubtarget::SOUTHERN_ISLANDS && " "GCNSubtarget::SOUTHERN_ISLANDS && "
"!AMDGPU::isShader(State.getCallingConv())", "!AMDGPU::isShader(State.getCallingConv())",
CCDelegateTo<CC_AMDGPU_Kernel>>, CCDelegateTo<CC_AMDGPU_Kernel>>,
CCIf<"static_cast<const AMDGPUSubtarget&>" CCIf<"static_cast<const GCNSubtarget&>"
"(State.getMachineFunction().getSubtarget()).getGeneration() >= " "(State.getMachineFunction().getSubtarget()).getGeneration() >= "
"AMDGPUSubtarget::SOUTHERN_ISLANDS", "GCNSubtarget::SOUTHERN_ISLANDS",
CCDelegateTo<CC_SI>>, CCDelegateTo<CC_SI>>,
CCIf<"static_cast<const AMDGPUSubtarget&>" CCIf<"static_cast<const GCNSubtarget&>"
"(State.getMachineFunction().getSubtarget()).getGeneration() >= " "(State.getMachineFunction().getSubtarget()).getGeneration() >= "
"AMDGPUSubtarget::SOUTHERN_ISLANDS && State.getCallingConv() == CallingConv::C", "GCNSubtarget::SOUTHERN_ISLANDS && State.getCallingConv() == CallingConv::C",
CCDelegateTo<CC_AMDGPU_Func>> CCDelegateTo<CC_AMDGPU_Func>>
]>; ]>;

lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

Show First 20 Lines • Show All 51 Lines • ▼ Show 20 Lines
static cl::opt<bool> WidenLoads( static cl::opt<bool> WidenLoads(
"amdgpu-codegenprepare-widen-constant-loads", "amdgpu-codegenprepare-widen-constant-loads",
cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"), cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"),
cl::ReallyHidden, cl::ReallyHidden,
cl::init(true)); cl::init(true));
class AMDGPUCodeGenPrepare : public FunctionPass, class AMDGPUCodeGenPrepare : public FunctionPass,
public InstVisitor<AMDGPUCodeGenPrepare, bool> { public InstVisitor<AMDGPUCodeGenPrepare, bool> {
const SISubtarget *ST = nullptr; const GCNSubtarget *ST = nullptr;
DivergenceAnalysis *DA = nullptr; DivergenceAnalysis *DA = nullptr;
Module *Mod = nullptr; Module *Mod = nullptr;
bool HasUnsafeFPMath = false; bool HasUnsafeFPMath = false;
AMDGPUAS AMDGPUASI; AMDGPUAS AMDGPUASI;
/// Copies exact/nsw/nuw flags (if any) from binary operation \p I to /// Copies exact/nsw/nuw flags (if any) from binary operation \p I to
/// binary operation \p V. /// binary operation \p V.
/// ///
▲ Show 20 Lines • Show All 816 Lines • ▼ Show 20 Linesbool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
if (skipFunction(F)) if (skipFunction(F))
return false; return false;
auto *TPC = getAnalysisIfAvailable<TargetPassConfig>(); auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
if (!TPC) if (!TPC)
return false; return false;
const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>(); const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
ST = &TM.getSubtarget<SISubtarget>(F); ST = &TM.getSubtarget<GCNSubtarget>(F);
DA = &getAnalysis<DivergenceAnalysis>(); DA = &getAnalysis<DivergenceAnalysis>();
HasUnsafeFPMath = hasUnsafeFPMath(F); HasUnsafeFPMath = hasUnsafeFPMath(F);
AMDGPUASI = TM.getAMDGPUAS(); AMDGPUASI = TM.getAMDGPUAS();
bool MadeChange = false; bool MadeChange = false;
for (BasicBlock &BB : F) { for (BasicBlock &BB : F) {
BasicBlock::iterator Next; BasicBlock::iterator Next;
Show All 20 Lines

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

Show First 20 Lines • Show All 65 Lines • ▼ Show 20 Lines
namespace { namespace {
/// AMDGPU specific code to select AMDGPU machine instructions for /// AMDGPU specific code to select AMDGPU machine instructions for
/// SelectionDAG operations. /// SelectionDAG operations.
class AMDGPUDAGToDAGISel : public SelectionDAGISel { class AMDGPUDAGToDAGISel : public SelectionDAGISel {
// Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
// make the right decision when generating code for different targets. // make the right decision when generating code for different targets.
const AMDGPUSubtarget *Subtarget; const GCNSubtarget *Subtarget;
AMDGPUAS AMDGPUASI; AMDGPUAS AMDGPUASI;
bool EnableLateStructurizeCFG; bool EnableLateStructurizeCFG;
public: public:
explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
CodeGenOpt::Level OptLevel = CodeGenOpt::Default) CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
: SelectionDAGISel(*TM, OptLevel) { : SelectionDAGISel(*TM, OptLevel) {
AMDGPUASI = AMDGPU::getAMDGPUAS(*TM); AMDGPUASI = AMDGPU::getAMDGPUAS(*TM);
▲ Show 20 Lines • Show All 186 Lines • ▼ Show 20 Lines
/// This pass converts a legalized DAG into a R600-specific /// This pass converts a legalized DAG into a R600-specific
// DAG, ready for instruction scheduling. // DAG, ready for instruction scheduling.
FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
CodeGenOpt::Level OptLevel) { CodeGenOpt::Level OptLevel) {
return new R600DAGToDAGISel(TM, OptLevel); return new R600DAGToDAGISel(TM, OptLevel);
} }
bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
Subtarget = &MF.getSubtarget<AMDGPUSubtarget>(); Subtarget = &MF.getSubtarget<GCNSubtarget>();
return SelectionDAGISel::runOnMachineFunction(MF); return SelectionDAGISel::runOnMachineFunction(MF);
} }
bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
if (TM.Options.NoNaNsFPMath) if (TM.Options.NoNaNsFPMath)
return true; return true;
// TODO: Move into isKnownNeverNaN // TODO: Move into isKnownNeverNaN
Show All 25 Lines if (!N->isMachineOpcode()) {
if (N->getOpcode() == ISD::CopyToReg) { if (N->getOpcode() == ISD::CopyToReg) {
unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
if (TargetRegisterInfo::isVirtualRegister(Reg)) { if (TargetRegisterInfo::isVirtualRegister(Reg)) {
MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
return MRI.getRegClass(Reg); return MRI.getRegClass(Reg);
} }
const SIRegisterInfo *TRI const SIRegisterInfo *TRI
= static_cast<const SISubtarget *>(Subtarget)->getRegisterInfo(); = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
return TRI->getPhysRegClass(Reg); return TRI->getPhysRegClass(Reg);
} }
return nullptr; return nullptr;
} }
switch (N->getMachineOpcode()) { switch (N->getMachineOpcode()) {
default: { default: {
▲ Show 20 Lines • Show All 493 Lines • ▼ Show 20 Lines
} }
bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset, bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
unsigned OffsetBits) const { unsigned OffsetBits) const {
if ((OffsetBits == 16 && !isUInt<16>(Offset)) || if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
(OffsetBits == 8 && !isUInt<8>(Offset))) (OffsetBits == 8 && !isUInt<8>(Offset)))
return false; return false;
if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS || if (Subtarget->getGeneration() >= GCNSubtarget::SEA_ISLANDS ||
Subtarget->unsafeDSOffsetFoldingEnabled()) Subtarget->unsafeDSOffsetFoldingEnabled())
return true; return true;
// On Southern Islands instruction with a negative base value and an offset // On Southern Islands instruction with a negative base value and an offset
// don't seem to work. // don't seem to work.
return CurDAG->SignBitIsZero(Base); return CurDAG->SignBitIsZero(Base);
} }
▲ Show 20 Lines • Show All 216 Lines • ▼ Show 20 Lines
bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
SDValue &VAddr, SDValue &SOffset, SDValue &VAddr, SDValue &SOffset,
SDValue &Offset, SDValue &GLC, SDValue &Offset, SDValue &GLC,
SDValue &SLC, SDValue &TFE) const { SDValue &SLC, SDValue &TFE) const {
SDValue Ptr, Offen, Idxen, Addr64; SDValue Ptr, Offen, Idxen, Addr64;
// addr64 bit was removed for volcanic islands. // addr64 bit was removed for volcanic islands.
if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) if (Subtarget->getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS)
return false; return false;
if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
GLC, SLC, TFE)) GLC, SLC, TFE))
return false; return false;
ConstantSDNode *C = cast<ConstantSDNode>(Addr64); ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
if (C->getSExtValue()) { if (C->getSExtValue()) {
▲ Show 20 Lines • Show All 211 Lines • ▼ Show 20 Lines if (Imm <= MaxImm + 64) {
Overflow = High - Align; Overflow = High - Align;
} }
} }
// There is a hardware bug in SI and CI which prevents address clamping in // There is a hardware bug in SI and CI which prevents address clamping in
// MUBUF instructions from working correctly with SOffsets. The immediate // MUBUF instructions from working correctly with SOffsets. The immediate
// offset is unaffected. // offset is unaffected.
if (Overflow > 0 && if (Overflow > 0 &&
Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) Subtarget->getGeneration() <= GCNSubtarget::SEA_ISLANDS)
return false; return false;
ImmOffset = CurDAG->getTargetConstant(Imm, DL, MVT::i16); ImmOffset = CurDAG->getTargetConstant(Imm, DL, MVT::i16);
if (Overflow <= 64) if (Overflow <= 64)
SOffset = CurDAG->getTargetConstant(Overflow, DL, MVT::i32); SOffset = CurDAG->getTargetConstant(Overflow, DL, MVT::i32);
else else
SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
Show All 21 Linesbool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicVOffset(SDValue Offset,
SDLoc DL(Offset); SDLoc DL(Offset);
// Don't generate an unnecessary voffset for constant offsets. // Don't generate an unnecessary voffset for constant offsets.
if (isa<ConstantSDNode>(Offset)) { if (isa<ConstantSDNode>(Offset)) {
SDValue Tmp1, Tmp2; SDValue Tmp1, Tmp2;
// When necessary, use a voffset in <= CI anyway to work around a hardware // When necessary, use a voffset in <= CI anyway to work around a hardware
// bug. // bug.
if (Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS || if (Subtarget->getGeneration() > GCNSubtarget::SEA_ISLANDS ||
SelectMUBUFConstant(Offset, Tmp1, Tmp2)) SelectMUBUFConstant(Offset, Tmp1, Tmp2))
return false; return false;
} }
if (CurDAG->isBaseWithConstantOffset(Offset)) { if (CurDAG->isBaseWithConstantOffset(Offset)) {
SDValue N0 = Offset.getOperand(0); SDValue N0 = Offset.getOperand(0);
SDValue N1 = Offset.getOperand(1); SDValue N1 = Offset.getOperand(1);
if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 && if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 &&
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Linesbool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
SDValue &Offset, bool &Imm) const { SDValue &Offset, bool &Imm) const {
// FIXME: Handle non-constant offsets. // FIXME: Handle non-constant offsets.
ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
if (!C) if (!C)
return false; return false;
SDLoc SL(ByteOffsetNode); SDLoc SL(ByteOffsetNode);
AMDGPUSubtarget::Generation Gen = Subtarget->getGeneration(); GCNSubtarget::Generation Gen = Subtarget->getGeneration();
int64_t ByteOffset = C->getSExtValue(); int64_t ByteOffset = C->getSExtValue();
int64_t EncodedOffset = AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset); int64_t EncodedOffset = AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset);
if (AMDGPU::isLegalSMRDImmOffset(*Subtarget, ByteOffset)) { if (AMDGPU::isLegalSMRDImmOffset(*Subtarget, ByteOffset)) {
Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32); Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
Imm = true; Imm = true;
return true; return true;
} }
if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset)) if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
return false; return false;
if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) { if (Gen == GCNSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
// 32-bit Immediates are supported on Sea Islands. // 32-bit Immediates are supported on Sea Islands.
Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32); Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
} else { } else {
SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
C32Bit), 0); C32Bit), 0);
} }
Imm = false; Imm = false;
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Linesbool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
SDValue &Offset) const { SDValue &Offset) const {
bool Imm; bool Imm;
return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
} }
bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
SDValue &Offset) const { SDValue &Offset) const {
if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS) if (Subtarget->getGeneration() != GCNSubtarget::SEA_ISLANDS)
return false; return false;
bool Imm; bool Imm;
if (!SelectSMRD(Addr, SBase, Offset, Imm)) if (!SelectSMRD(Addr, SBase, Offset, Imm))
return false; return false;
return !Imm && isa<ConstantSDNode>(Offset); return !Imm && isa<ConstantSDNode>(Offset);
} }
bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
SDValue &Offset) const { SDValue &Offset) const {
bool Imm; bool Imm;
return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
!isa<ConstantSDNode>(Offset); !isa<ConstantSDNode>(Offset);
} }
bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
SDValue &Offset) const { SDValue &Offset) const {
bool Imm; bool Imm;
return SelectSMRDOffset(Addr, Offset, Imm) && Imm; return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
} }
bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
SDValue &Offset) const { SDValue &Offset) const {
if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS) if (Subtarget->getGeneration() != GCNSubtarget::SEA_ISLANDS)
return false; return false;
bool Imm; bool Imm;
if (!SelectSMRDOffset(Addr, Offset, Imm)) if (!SelectSMRDOffset(Addr, Offset, Imm))
return false; return false;
return !Imm && isa<ConstantSDNode>(Offset); return !Imm && isa<ConstantSDNode>(Offset);
} }
▲ Show 20 Lines • Show All 147 Lines • ▼ Show 20 Linesbool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
return false; return false;
MVT VT = Cond.getOperand(0).getSimpleValueType(); MVT VT = Cond.getOperand(0).getSimpleValueType();
if (VT == MVT::i32) if (VT == MVT::i32)
return true; return true;
if (VT == MVT::i64) { if (VT == MVT::i64) {
auto ST = static_cast<const SISubtarget *>(Subtarget); auto ST = static_cast<const GCNSubtarget *>(Subtarget);
ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
} }
return false; return false;
} }
▲ Show 20 Lines • Show All 590 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUISelLowering.h

Show All 17 Lines
#include "AMDGPU.h" #include "AMDGPU.h"
#include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetLowering.h"
namespace llvm { namespace llvm {
class AMDGPUMachineFunction; class AMDGPUMachineFunction;
class AMDGPUCommonSubtarget; class AMDGPUSubtarget;
struct ArgDescriptor; struct ArgDescriptor;
class AMDGPUTargetLowering : public TargetLowering { class AMDGPUTargetLowering : public TargetLowering {
private: private:
const AMDGPUCommonSubtarget *Subtarget; const AMDGPUSubtarget *Subtarget;
/// \returns AMDGPUISD::FFBH_U32 node if the incoming \p Op may have been /// \returns AMDGPUISD::FFBH_U32 node if the incoming \p Op may have been
/// legalized from a smaller type VT. Need to match pre-legalized type because /// legalized from a smaller type VT. Need to match pre-legalized type because
/// the generic legalization inserts the add/sub between the select and /// the generic legalization inserts the add/sub between the select and
/// compare. /// compare.
SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL, unsigned Opc) const; SDValue getFFBX_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL, unsigned Opc) const;
public: public:
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Linesprotected:
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const; SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const; SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const;
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG,
SmallVectorImpl<SDValue> &Results) const; SmallVectorImpl<SDValue> &Results) const;
void analyzeFormalArgumentsCompute(CCState &State, void analyzeFormalArgumentsCompute(CCState &State,
const SmallVectorImpl<ISD::InputArg> &Ins) const; const SmallVectorImpl<ISD::InputArg> &Ins) const;
public: public:
AMDGPUTargetLowering(const TargetMachine &TM, const AMDGPUCommonSubtarget &STI); AMDGPUTargetLowering(const TargetMachine &TM, const AMDGPUSubtarget &STI);
bool mayIgnoreSignedZero(SDValue Op) const { bool mayIgnoreSignedZero(SDValue Op) const {
if (getTargetMachine().Options.NoSignedZerosFPMath) if (getTargetMachine().Options.NoSignedZerosFPMath)
return true; return true;
const auto Flags = Op.getNode()->getFlags(); const auto Flags = Op.getNode()->getFlags();
if (Flags.isDefined()) if (Flags.isDefined())
return Flags.hasNoSignedZeros(); return Flags.hasNoSignedZeros();
▲ Show 20 Lines • Show All 365 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

Show First 20 Lines • Show All 149 Lines • ▼ Show 20 Linesunsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
EVT VT = Op.getValueType(); EVT VT = Op.getValueType();
// In order for this to be a signed 24-bit value, bit 23, must // In order for this to be a signed 24-bit value, bit 23, must
// be a sign bit. // be a sign bit.
return VT.getSizeInBits() - DAG.ComputeNumSignBits(Op); return VT.getSizeInBits() - DAG.ComputeNumSignBits(Op);
} }
AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
const AMDGPUCommonSubtarget &STI) const AMDGPUSubtarget &STI)
: TargetLowering(TM), Subtarget(&STI) { : TargetLowering(TM), Subtarget(&STI) {
AMDGPUASI = AMDGPU::getAMDGPUAS(TM); AMDGPUASI = AMDGPU::getAMDGPUAS(TM);
// Lower floating point store/load to integer store/load to reduce the number // Lower floating point store/load to integer store/load to reduce the number
// of patterns in tablegen. // of patterns in tablegen.
setOperationAction(ISD::LOAD, MVT::f32, Promote); setOperationAction(ISD::LOAD, MVT::f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);
setOperationAction(ISD::LOAD, MVT::v2f32, Promote); setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
▲ Show 20 Lines • Show All 3,767 Lines • ▼ Show 20 LinesSDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
if (Arg.isRegister()) if (Arg.isRegister())
return CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL); return CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL);
return loadStackInputValue(DAG, VT, SL, Arg.getStackOffset()); return loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
} }
uint32_t AMDGPUTargetLowering::getImplicitParameterOffset( uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
const MachineFunction &MF, const ImplicitParameter Param) const { const MachineFunction &MF, const ImplicitParameter Param) const {
const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>(); const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
const AMDGPUCommonSubtarget &ST = const AMDGPUSubtarget &ST =
AMDGPUCommonSubtarget::get(getTargetMachine(), MF.getFunction()); AMDGPUSubtarget::get(getTargetMachine(), MF.getFunction());
unsigned ExplicitArgOffset = ST.getExplicitKernelArgOffset(MF.getFunction()); unsigned ExplicitArgOffset = ST.getExplicitKernelArgOffset(MF.getFunction());
unsigned Alignment = ST.getAlignmentForImplicitArgPtr(); unsigned Alignment = ST.getAlignmentForImplicitArgPtr();
uint64_t ArgOffset = alignTo(MFI->getExplicitKernArgSize(), Alignment) + uint64_t ArgOffset = alignTo(MFI->getExplicitKernArgSize(), Alignment) +
ExplicitArgOffset; ExplicitArgOffset;
switch (Param) { switch (Param) {
case GRID_DIM: case GRID_DIM:
return ArgOffset; return ArgOffset;
case GRID_OFFSET: case GRID_OFFSET:
▲ Show 20 Lines • Show All 285 Lines • ▼ Show 20 Lines case AMDGPUISD::PERM: {
} }
break; break;
} }
case ISD::INTRINSIC_WO_CHAIN: { case ISD::INTRINSIC_WO_CHAIN: {
unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
switch (IID) { switch (IID) {
case Intrinsic::amdgcn_mbcnt_lo: case Intrinsic::amdgcn_mbcnt_lo:
case Intrinsic::amdgcn_mbcnt_hi: { case Intrinsic::amdgcn_mbcnt_hi: {
const SISubtarget &ST = const GCNSubtarget &ST =
DAG.getMachineFunction().getSubtarget<SISubtarget>(); DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
// These return at most the wavefront size - 1. // These return at most the wavefront size - 1.
unsigned Size = Op.getValueType().getSizeInBits(); unsigned Size = Op.getValueType().getSizeInBits();
Known.Zero.setHighBits(Size - ST.getWavefrontSizeLog2()); Known.Zero.setHighBits(Size - ST.getWavefrontSizeLog2());
break; break;
} }
default: default:
break; break;
} }
Show All 37 Lines

lib/Target/AMDGPU/AMDGPUInstrInfo.h

Show All 16 Lines
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRINFO_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRINFO_H
#include "AMDGPU.h" #include "AMDGPU.h"
#include "Utils/AMDGPUBaseInfo.h" #include "Utils/AMDGPUBaseInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h"
namespace llvm { namespace llvm {
class AMDGPUSubtarget; class GCNSubtarget;
class MachineFunction; class MachineFunction;
class MachineInstr; class MachineInstr;
class MachineInstrBuilder; class MachineInstrBuilder;
class AMDGPUInstrInfo { class AMDGPUInstrInfo {
public: public:
explicit AMDGPUInstrInfo(const AMDGPUSubtarget &st); explicit AMDGPUInstrInfo(const GCNSubtarget &st);
static bool isUniformMMO(const MachineMemOperand *MMO); static bool isUniformMMO(const MachineMemOperand *MMO);
}; };
namespace AMDGPU { namespace AMDGPU {
struct RsrcIntrinsic { struct RsrcIntrinsic {
unsigned Intr; unsigned Intr;
Show All 22 Lines

lib/Target/AMDGPU/AMDGPUInstrInfo.cpp

Show All 20 Lines
#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h"
using namespace llvm; using namespace llvm;
// Pin the vtable to this file. // Pin the vtable to this file.
//void AMDGPUInstrInfo::anchor() {} //void AMDGPUInstrInfo::anchor() {}
AMDGPUInstrInfo::AMDGPUInstrInfo(const AMDGPUSubtarget &ST) { } AMDGPUInstrInfo::AMDGPUInstrInfo(const GCNSubtarget &ST) { }
// TODO: Should largely merge with AMDGPUTTIImpl::isSourceOfDivergence. // TODO: Should largely merge with AMDGPUTTIImpl::isSourceOfDivergence.
bool AMDGPUInstrInfo::isUniformMMO(const MachineMemOperand *MMO) { bool AMDGPUInstrInfo::isUniformMMO(const MachineMemOperand *MMO) {
const Value *Ptr = MMO->getValue(); const Value *Ptr = MMO->getValue();
// UndefValue means this is a load of a kernel input. These are uniform. // UndefValue means this is a load of a kernel input. These are uniform.
// Sometimes LDS instructions have constant pointers. // Sometimes LDS instructions have constant pointers.
// If Ptr is null, then that means this mem operand contains a // If Ptr is null, then that means this mem operand contains a
Show All 14 Lines

lib/Target/AMDGPU/AMDGPUInstructionSelector.h

Show All 16 Lines
#include "AMDGPU.h" #include "AMDGPU.h"
#include "AMDGPUArgumentUsageInfo.h" #include "AMDGPUArgumentUsageInfo.h"
#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
namespace { namespace {
#define GET_GLOBALISEL_PREDICATE_BITSET #define GET_GLOBALISEL_PREDICATE_BITSET
#define AMDGPUSubtarget GCNSubtarget
#include "AMDGPUGenGlobalISel.inc" #include "AMDGPUGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATE_BITSET #undef GET_GLOBALISEL_PREDICATE_BITSET
#undef AMDGPUSubtarget
} }
namespace llvm { namespace llvm {
class AMDGPUInstrInfo; class AMDGPUInstrInfo;
class AMDGPURegisterBankInfo; class AMDGPURegisterBankInfo;
class AMDGPUSubtarget; class GCNSubtarget;
class MachineInstr; class MachineInstr;
class MachineOperand; class MachineOperand;
class MachineRegisterInfo; class MachineRegisterInfo;
class SIInstrInfo; class SIInstrInfo;
class SIMachineFunctionInfo; class SIMachineFunctionInfo;
class SIRegisterInfo; class SIRegisterInfo;
class SISubtarget; class GCNSubtarget;
jveselyUnsubmitted
Done
ReplyInline Actions

This rename duplicates GCNSubtarget declaration.

jvesely: This rename duplicates GCNSubtarget declaration.
class AMDGPUInstructionSelector : public InstructionSelector { class AMDGPUInstructionSelector : public InstructionSelector {
public: public:
AMDGPUInstructionSelector(const SISubtarget &STI, AMDGPUInstructionSelector(const GCNSubtarget &STI,
const AMDGPURegisterBankInfo &RBI, const AMDGPURegisterBankInfo &RBI,
const AMDGPUTargetMachine &TM); const AMDGPUTargetMachine &TM);
bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override; bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
static const char *getName(); static const char *getName();
private: private:
struct GEPInfo { struct GEPInfo {
Show All 33 Linesprivate:
selectVOP3OMods(MachineOperand &Root) const; selectVOP3OMods(MachineOperand &Root) const;
InstructionSelector::ComplexRendererFns InstructionSelector::ComplexRendererFns
selectVOP3Mods(MachineOperand &Root) const; selectVOP3Mods(MachineOperand &Root) const;
const SIInstrInfo &TII; const SIInstrInfo &TII;
const SIRegisterInfo &TRI; const SIRegisterInfo &TRI;
const AMDGPURegisterBankInfo &RBI; const AMDGPURegisterBankInfo &RBI;
const AMDGPUTargetMachine &TM; const AMDGPUTargetMachine &TM;
const SISubtarget &STI; const GCNSubtarget &STI;
bool EnableLateStructurizeCFG; bool EnableLateStructurizeCFG;
#define GET_GLOBALISEL_PREDICATES_DECL #define GET_GLOBALISEL_PREDICATES_DECL
#define AMDGPUSubtarget GCNSubtarget
#include "AMDGPUGenGlobalISel.inc" #include "AMDGPUGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_DECL #undef GET_GLOBALISEL_PREDICATES_DECL
#undef AMDGPUSubtarget
#define GET_GLOBALISEL_TEMPORARIES_DECL #define GET_GLOBALISEL_TEMPORARIES_DECL
#include "AMDGPUGenGlobalISel.inc" #include "AMDGPUGenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_DECL #undef GET_GLOBALISEL_TEMPORARIES_DECL
protected: protected:
AMDGPUAS AMDGPUASI; AMDGPUAS AMDGPUASI;
}; };
} // End llvm namespace. } // End llvm namespace.
#endif #endif

lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

Show All 31 Lines
#include "llvm/Support/Debug.h" #include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h" #include "llvm/Support/raw_ostream.h"
#define DEBUG_TYPE "amdgpu-isel" #define DEBUG_TYPE "amdgpu-isel"
using namespace llvm; using namespace llvm;
#define GET_GLOBALISEL_IMPL #define GET_GLOBALISEL_IMPL
#define AMDGPUSubtarget GCNSubtarget
#include "AMDGPUGenGlobalISel.inc" #include "AMDGPUGenGlobalISel.inc"
#undef GET_GLOBALISEL_IMPL #undef GET_GLOBALISEL_IMPL
#undef AMDGPUSubtarget
AMDGPUInstructionSelector::AMDGPUInstructionSelector( AMDGPUInstructionSelector::AMDGPUInstructionSelector(
const SISubtarget &STI, const AMDGPURegisterBankInfo &RBI, const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI,
const AMDGPUTargetMachine &TM) const AMDGPUTargetMachine &TM)
: InstructionSelector(), TII(*STI.getInstrInfo()), : InstructionSelector(), TII(*STI.getInstrInfo()),
TRI(*STI.getRegisterInfo()), RBI(RBI), TM(TM), TRI(*STI.getRegisterInfo()), RBI(RBI), TM(TM),
STI(STI), STI(STI),
EnableLateStructurizeCFG(AMDGPUTargetMachine::EnableLateStructurizeCFG), EnableLateStructurizeCFG(AMDGPUTargetMachine::EnableLateStructurizeCFG),
#define GET_GLOBALISEL_PREDICATES_INIT #define GET_GLOBALISEL_PREDICATES_INIT
#include "AMDGPUGenGlobalISel.inc" #include "AMDGPUGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_INIT #undef GET_GLOBALISEL_PREDICATES_INIT
▲ Show 20 Lines • Show All 389 Lines • ▼ Show 20 Linesbool AMDGPUInstructionSelector::selectSMRD(MachineInstr &I,
if (!isInstrUniform(I)) if (!isInstrUniform(I))
return false; return false;
if (hasVgprParts(AddrInfo)) if (hasVgprParts(AddrInfo))
return false; return false;
MachineBasicBlock *BB = I.getParent(); MachineBasicBlock *BB = I.getParent();
MachineFunction *MF = BB->getParent(); MachineFunction *MF = BB->getParent();
const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>();
MachineRegisterInfo &MRI = MF->getRegInfo(); MachineRegisterInfo &MRI = MF->getRegInfo();
unsigned DstReg = I.getOperand(0).getReg(); unsigned DstReg = I.getOperand(0).getReg();
const DebugLoc &DL = I.getDebugLoc(); const DebugLoc &DL = I.getDebugLoc();
unsigned Opcode; unsigned Opcode;
unsigned LoadSize = RBI.getSizeInBits(DstReg, MRI, TRI); unsigned LoadSize = RBI.getSizeInBits(DstReg, MRI, TRI);
if (!AddrInfo.empty() && AddrInfo[0].SgprParts.size() == 1) { if (!AddrInfo.empty() && AddrInfo[0].SgprParts.size() == 1) {
const GEPInfo &GEPInfo = AddrInfo[0]; const GEPInfo &GEPInfo = AddrInfo[0];
unsigned PtrReg = GEPInfo.SgprParts[0]; unsigned PtrReg = GEPInfo.SgprParts[0];
int64_t EncodedImm = AMDGPU::getSMRDEncodedOffset(Subtarget, GEPInfo.Imm); int64_t EncodedImm = AMDGPU::getSMRDEncodedOffset(Subtarget, GEPInfo.Imm);
if (AMDGPU::isLegalSMRDImmOffset(Subtarget, GEPInfo.Imm)) { if (AMDGPU::isLegalSMRDImmOffset(Subtarget, GEPInfo.Imm)) {
Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM, LoadSize); Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM, LoadSize);
MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg) MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
.addReg(PtrReg) .addReg(PtrReg)
.addImm(EncodedImm) .addImm(EncodedImm)
.addImm(0); // glc .addImm(0); // glc
return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI); return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
} }
if (Subtarget.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS && if (Subtarget.getGeneration() == GCNSubtarget::SEA_ISLANDS &&
isUInt<32>(EncodedImm)) { isUInt<32>(EncodedImm)) {
Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM_ci, LoadSize); Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM_ci, LoadSize);
MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg) MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
.addReg(PtrReg) .addReg(PtrReg)
.addImm(EncodedImm) .addImm(EncodedImm)
.addImm(0); // glc .addImm(0); // glc
return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI); return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
} }
▲ Show 20 Lines • Show All 143 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPULegalizerInfo.h

Show All 15 Lines
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINELEGALIZER_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINELEGALIZER_H
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
namespace llvm { namespace llvm {
class GCNTargetMachine; class GCNTargetMachine;
class LLVMContext; class LLVMContext;
class SISubtarget; class GCNSubtarget;
/// This class provides the information for the target register banks. /// This class provides the information for the target register banks.
class AMDGPULegalizerInfo : public LegalizerInfo { class AMDGPULegalizerInfo : public LegalizerInfo {
public: public:
AMDGPULegalizerInfo(const SISubtarget &ST, AMDGPULegalizerInfo(const GCNSubtarget &ST,
const GCNTargetMachine &TM); const GCNTargetMachine &TM);
}; };
} // End llvm namespace. } // End llvm namespace.
#endif #endif

lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Show All 18 Lines
#include "llvm/CodeGen/ValueTypes.h" #include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Type.h" #include "llvm/IR/Type.h"
#include "llvm/Support/Debug.h" #include "llvm/Support/Debug.h"
using namespace llvm; using namespace llvm;
using namespace LegalizeActions; using namespace LegalizeActions;
AMDGPULegalizerInfo::AMDGPULegalizerInfo(const SISubtarget &ST, AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST,
const GCNTargetMachine &TM) { const GCNTargetMachine &TM) {
using namespace TargetOpcode; using namespace TargetOpcode;
auto GetAddrSpacePtr = [&TM](unsigned AS) { auto GetAddrSpacePtr = [&TM](unsigned AS) {
return LLT::pointer(AS, TM.getPointerSizeInBits(AS)); return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
}; };
auto AMDGPUAS = ST.getAMDGPUAS(); auto AMDGPUAS = ST.getAMDGPUAS();
▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines .legalIf([=, &ST](const LegalityQuery &Query) {
switch (Ty0.getSizeInBits()) { switch (Ty0.getSizeInBits()) {
case 32: case 32:
case 64: case 64:
case 128: case 128:
return true; return true;
case 96: case 96:
// XXX hasLoadX3 // XXX hasLoadX3
return (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS); return (ST.getGeneration() >= GCNSubtarget::SEA_ISLANDS);
case 256: case 256:
case 512: case 512:
// TODO: constant loads // TODO: constant loads
default: default:
return false; return false;
} }
}); });
▲ Show 20 Lines • Show All 70 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp

Show First 20 Lines • Show All 118 Lines • ▼ Show 20 Linesbool AMDGPULowerIntrinsics::makeLIDRangeMetadata(Function &F) const {
const TargetMachine &TM = TPC->getTM<TargetMachine>(); const TargetMachine &TM = TPC->getTM<TargetMachine>();
bool Changed = false; bool Changed = false;
for (auto *U : F.users()) { for (auto *U : F.users()) {
auto *CI = dyn_cast<CallInst>(U); auto *CI = dyn_cast<CallInst>(U);
if (!CI) if (!CI)
continue; continue;
Changed |= AMDGPUCommonSubtarget::get(TM, F).makeLIDRangeMetadata(CI); Changed |= AMDGPUSubtarget::get(TM, F).makeLIDRangeMetadata(CI);
} }
return Changed; return Changed;
} }
bool AMDGPULowerIntrinsics::runOnModule(Module &M) { bool AMDGPULowerIntrinsics::runOnModule(Module &M) {
bool Changed = false; bool Changed = false;
for (Function &F : M) { for (Function &F : M) {
Show All 34 Lines

lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp

Show First 20 Lines • Show All 62 Lines • ▼ Show 20 Lines
bool AMDGPULowerKernelArguments::runOnFunction(Function &F) { bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
CallingConv::ID CC = F.getCallingConv(); CallingConv::ID CC = F.getCallingConv();
if (CC != CallingConv::AMDGPU_KERNEL || F.arg_empty()) if (CC != CallingConv::AMDGPU_KERNEL || F.arg_empty())
return false; return false;
auto &TPC = getAnalysis<TargetPassConfig>(); auto &TPC = getAnalysis<TargetPassConfig>();
const TargetMachine &TM = TPC.getTM<TargetMachine>(); const TargetMachine &TM = TPC.getTM<TargetMachine>();
const SISubtarget &ST = TM.getSubtarget<SISubtarget>(F); const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
LLVMContext &Ctx = F.getParent()->getContext(); LLVMContext &Ctx = F.getParent()->getContext();
const DataLayout &DL = F.getParent()->getDataLayout(); const DataLayout &DL = F.getParent()->getDataLayout();
BasicBlock &EntryBlock = *F.begin(); BasicBlock &EntryBlock = *F.begin();
IRBuilder<> Builder(&*EntryBlock.begin()); IRBuilder<> Builder(&*EntryBlock.begin());
const unsigned KernArgBaseAlign = 16; // FIXME: Increase if necessary const unsigned KernArgBaseAlign = 16; // FIXME: Increase if necessary
const uint64_t BaseOffset = ST.getExplicitKernelArgOffset(F); const uint64_t BaseOffset = ST.getExplicitKernelArgOffset(F);
Show All 38 Lines if (Arg.use_empty())
continue; continue;
if (PointerType *PT = dyn_cast<PointerType>(ArgTy)) { if (PointerType *PT = dyn_cast<PointerType>(ArgTy)) {
// FIXME: Hack. We rely on AssertZext to be able to fold DS addressing // FIXME: Hack. We rely on AssertZext to be able to fold DS addressing
// modes on SI to know the high bits are 0 so pointer adds don't wrap. We // modes on SI to know the high bits are 0 so pointer adds don't wrap. We
// can't represent this with range metadata because it's only allowed for // can't represent this with range metadata because it's only allowed for
// integer types. // integer types.
if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS && if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
ST.getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) ST.getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS)
continue; continue;
// FIXME: We can replace this with equivalent alias.scope/noalias // FIXME: We can replace this with equivalent alias.scope/noalias
// metadata, but this appears to be a lot of work. // metadata, but this appears to be a lot of work.
if (Arg.hasNoAliasAttr()) if (Arg.hasNoAliasAttr())
continue; continue;
} }
▲ Show 20 Lines • Show All 131 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUMCInstLower.cpp

Show First 20 Lines • Show All 198 Lines • ▼ Show 20 Lines for (const MachineOperand &MO : MI->explicit_operands()) {
MCOperand MCOp; MCOperand MCOp;
lowerOperand(MO, MCOp); lowerOperand(MO, MCOp);
OutMI.addOperand(MCOp); OutMI.addOperand(MCOp);
} }
} }
bool AMDGPUAsmPrinter::lowerOperand(const MachineOperand &MO, bool AMDGPUAsmPrinter::lowerOperand(const MachineOperand &MO,
MCOperand &MCOp) const { MCOperand &MCOp) const {
const AMDGPUSubtarget &STI = MF->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();
AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this); AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this);
return MCInstLowering.lowerOperand(MO, MCOp); return MCInstLowering.lowerOperand(MO, MCOp);
} }
static const MCExpr *lowerAddrSpaceCast(const TargetMachine &TM, static const MCExpr *lowerAddrSpaceCast(const TargetMachine &TM,
const Constant *CV, const Constant *CV,
MCContext &OutContext) { MCContext &OutContext) {
// TargetMachine does not support llvm-style cast. Use C++-style cast. // TargetMachine does not support llvm-style cast. Use C++-style cast.
Show All 22 Lines if (const MCExpr *E = lowerAddrSpaceCast(TM, CV, OutContext))
return E; return E;
return AsmPrinter::lowerConstant(CV); return AsmPrinter::lowerConstant(CV);
} }
void AMDGPUAsmPrinter::EmitInstruction(const MachineInstr *MI) { void AMDGPUAsmPrinter::EmitInstruction(const MachineInstr *MI) {
if (emitPseudoExpansionLowering(*OutStreamer, MI)) if (emitPseudoExpansionLowering(*OutStreamer, MI))
return; return;
const AMDGPUSubtarget &STI = MF->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();
AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this); AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this);
StringRef Err; StringRef Err;
if (!STI.getInstrInfo()->verifyInstruction(*MI, Err)) { if (!STI.getInstrInfo()->verifyInstruction(*MI, Err)) {
LLVMContext &C = MI->getParent()->getParent()->getFunction().getContext(); LLVMContext &C = MI->getParent()->getParent()->getFunction().getContext();
C.emitError("Illegal instruction detected: " + Err); C.emitError("Illegal instruction detected: " + Err);
MI->print(errs()); MI->print(errs());
} }
▲ Show 20 Lines • Show All 127 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp

Show First 20 Lines • Show All 2,865 Lines • ▼ Show 20 Lines for (MachineBasicBlock::instr_iterator I = MBBI.instr_begin(),
"PHI Operand not a register"); "PHI Operand not a register");
} }
} }
} }
} }
} }
bool AMDGPUMachineCFGStructurizer::runOnMachineFunction(MachineFunction &MF) { bool AMDGPUMachineCFGStructurizer::runOnMachineFunction(MachineFunction &MF) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
TRI = ST.getRegisterInfo(); TRI = ST.getRegisterInfo();
MRI = &(MF.getRegInfo()); MRI = &(MF.getRegInfo());
initFallthroughMap(MF); initFallthroughMap(MF);
checkRegOnlyPHIInputs(MF); checkRegOnlyPHIInputs(MF);
LLVM_DEBUG(dbgs() << "----STRUCTURIZER START----\n"); LLVM_DEBUG(dbgs() << "----STRUCTURIZER START----\n");
LLVM_DEBUG(MF.dump()); LLVM_DEBUG(MF.dump());
Show All 26 Lines

lib/Target/AMDGPU/AMDGPUMachineFunction.h

Show All 9 Lines
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEFUNCTION_H #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEFUNCTION_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEFUNCTION_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUMACHINEFUNCTION_H
#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMap.h"
#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunction.h"
namespace llvm { namespace llvm {
class AMDGPUSubtarget; class GCNSubtarget;
class AMDGPUMachineFunction : public MachineFunctionInfo { class AMDGPUMachineFunction : public MachineFunctionInfo {
/// A map to keep track of local memory objects and their offsets within the /// A map to keep track of local memory objects and their offsets within the
/// local memory space. /// local memory space.
SmallDenseMap<const GlobalValue *, unsigned, 4> LocalMemoryObjects; SmallDenseMap<const GlobalValue *, unsigned, 4> LocalMemoryObjects;
protected: protected:
uint64_t ExplicitKernArgSize; uint64_t ExplicitKernArgSize;
▲ Show 20 Lines • Show All 64 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp

Show First 20 Lines • Show All 146 Lines • ▼ Show 20 Lines if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>())
TM = &TPC->getTM<TargetMachine>(); TM = &TPC->getTM<TargetMachine>();
else else
return false; return false;
const Triple &TT = TM->getTargetTriple(); const Triple &TT = TM->getTargetTriple();
IsAMDGCN = TT.getArch() == Triple::amdgcn; IsAMDGCN = TT.getArch() == Triple::amdgcn;
IsAMDHSA = TT.getOS() == Triple::AMDHSA; IsAMDHSA = TT.getOS() == Triple::AMDHSA;
const AMDGPUCommonSubtarget &ST = AMDGPUCommonSubtarget::get(*TM, F); const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, F);
if (!ST.isPromoteAllocaEnabled()) if (!ST.isPromoteAllocaEnabled())
return false; return false;
AS = AMDGPU::getAMDGPUAS(*F.getParent()); AS = AMDGPU::getAMDGPUAS(*F.getParent());
bool SufficientLDS = hasSufficientLocalMem(F); bool SufficientLDS = hasSufficientLocalMem(F);
bool Changed = false; bool Changed = false;
BasicBlock &EntryBB = *F.begin(); BasicBlock &EntryBB = *F.begin();
for (auto I = EntryBB.begin(), E = EntryBB.end(); I != E; ) { for (auto I = EntryBB.begin(), E = EntryBB.end(); I != E; ) {
AllocaInst *AI = dyn_cast<AllocaInst>(I); AllocaInst *AI = dyn_cast<AllocaInst>(I);
++I; ++I;
if (AI) if (AI)
Changed |= handleAlloca(*AI, SufficientLDS); Changed |= handleAlloca(*AI, SufficientLDS);
} }
return Changed; return Changed;
} }
std::pair<Value *, Value *> std::pair<Value *, Value *>
AMDGPUPromoteAlloca::getLocalSizeYZ(IRBuilder<> &Builder) { AMDGPUPromoteAlloca::getLocalSizeYZ(IRBuilder<> &Builder) {
const Function &F = *Builder.GetInsertBlock()->getParent(); const Function &F = *Builder.GetInsertBlock()->getParent();
const AMDGPUCommonSubtarget &ST = AMDGPUCommonSubtarget::get(*TM, F); const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, F);
if (!IsAMDHSA) { if (!IsAMDHSA) {
Function *LocalSizeYFn Function *LocalSizeYFn
= Intrinsic::getDeclaration(Mod, Intrinsic::r600_read_local_size_y); = Intrinsic::getDeclaration(Mod, Intrinsic::r600_read_local_size_y);
Function *LocalSizeZFn Function *LocalSizeZFn
= Intrinsic::getDeclaration(Mod, Intrinsic::r600_read_local_size_z); = Intrinsic::getDeclaration(Mod, Intrinsic::r600_read_local_size_z);
CallInst *LocalSizeY = Builder.CreateCall(LocalSizeYFn, {}); CallInst *LocalSizeY = Builder.CreateCall(LocalSizeYFn, {});
▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 LinesAMDGPUPromoteAlloca::getLocalSizeYZ(IRBuilder<> &Builder) {
// Extract y component. Upper half of LoadZU should be zero already. // Extract y component. Upper half of LoadZU should be zero already.
Value *Y = Builder.CreateLShr(LoadXY, 16); Value *Y = Builder.CreateLShr(LoadXY, 16);
return std::make_pair(Y, LoadZU); return std::make_pair(Y, LoadZU);
} }
Value *AMDGPUPromoteAlloca::getWorkitemID(IRBuilder<> &Builder, unsigned N) { Value *AMDGPUPromoteAlloca::getWorkitemID(IRBuilder<> &Builder, unsigned N) {
const AMDGPUCommonSubtarget &ST = const AMDGPUSubtarget &ST =
AMDGPUCommonSubtarget::get(*TM, *Builder.GetInsertBlock()->getParent()); AMDGPUSubtarget::get(*TM, *Builder.GetInsertBlock()->getParent());
Intrinsic::ID IntrID = Intrinsic::ID::not_intrinsic; Intrinsic::ID IntrID = Intrinsic::ID::not_intrinsic;
switch (N) { switch (N) {
case 0: case 0:
IntrID = IsAMDGCN ? Intrinsic::amdgcn_workitem_id_x IntrID = IsAMDGCN ? Intrinsic::amdgcn_workitem_id_x
: Intrinsic::r600_read_tidig_x; : Intrinsic::r600_read_tidig_x;
break; break;
case 1: case 1:
▲ Show 20 Lines • Show All 324 Lines • ▼ Show 20 Linesbool AMDGPUPromoteAlloca::collectUsesWithPtrTypes(
} }
return true; return true;
} }
bool AMDGPUPromoteAlloca::hasSufficientLocalMem(const Function &F) { bool AMDGPUPromoteAlloca::hasSufficientLocalMem(const Function &F) {
FunctionType *FTy = F.getFunctionType(); FunctionType *FTy = F.getFunctionType();
const AMDGPUCommonSubtarget &ST = AMDGPUCommonSubtarget::get(*TM, F); const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, F);
// If the function has any arguments in the local address space, then it's // If the function has any arguments in the local address space, then it's
// possible these arguments require the entire local memory space, so // possible these arguments require the entire local memory space, so
// we cannot use local memory in the pass. // we cannot use local memory in the pass.
for (Type *ParamTy : FTy->params()) { for (Type *ParamTy : FTy->params()) {
PointerType *PtrTy = dyn_cast<PointerType>(ParamTy); PointerType *PtrTy = dyn_cast<PointerType>(ParamTy);
if (PtrTy && PtrTy->getAddressSpace() == AS.LOCAL_ADDRESS) { if (PtrTy && PtrTy->getAddressSpace() == AS.LOCAL_ADDRESS) {
LocalMemLimit = 0; LocalMemLimit = 0;
▲ Show 20 Lines • Show All 110 Lines • ▼ Show 20 Lines LLVM_DEBUG(
<< " promote alloca to LDS not supported with calling convention.\n"); << " promote alloca to LDS not supported with calling convention.\n");
return false; return false;
} }
// Not likely to have sufficient local memory for promotion. // Not likely to have sufficient local memory for promotion.
if (!SufficientLDS) if (!SufficientLDS)
return false; return false;
const AMDGPUCommonSubtarget &ST = AMDGPUCommonSubtarget::get(*TM, ContainingFunction); const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, ContainingFunction);
unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(ContainingFunction).second; unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(ContainingFunction).second;
const DataLayout &DL = Mod->getDataLayout(); const DataLayout &DL = Mod->getDataLayout();
unsigned Align = I.getAlignment(); unsigned Align = I.getAlignment();
if (Align == 0) if (Align == 0)
Align = DL.getABITypeAlignment(I.getAllocatedType()); Align = DL.getABITypeAlignment(I.getAllocatedType());
▲ Show 20 Lines • Show All 175 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPURegisterInfo.h

Show All 15 Lines
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUREGISTERINFO_H #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUREGISTERINFO_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUREGISTERINFO_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUREGISTERINFO_H
#define GET_REGINFO_HEADER #define GET_REGINFO_HEADER
#include "AMDGPUGenRegisterInfo.inc" #include "AMDGPUGenRegisterInfo.inc"
namespace llvm { namespace llvm {
class AMDGPUSubtarget; class GCNSubtarget;
class TargetInstrInfo; class TargetInstrInfo;
struct AMDGPURegisterInfo : public AMDGPUGenRegisterInfo { struct AMDGPURegisterInfo : public AMDGPUGenRegisterInfo {
AMDGPURegisterInfo(); AMDGPURegisterInfo();
bool enableMultipleCopyHints() const override { return true; } bool enableMultipleCopyHints() const override { return true; }
/// \returns the sub reg enum value for the given \p Channel /// \returns the sub reg enum value for the given \p Channel
Show All 9 Lines

lib/Target/AMDGPU/AMDGPUSubtarget.h

Show All 40 Lines
#include "AMDGPUGenSubtargetInfo.inc" #include "AMDGPUGenSubtargetInfo.inc"
#define GET_SUBTARGETINFO_HEADER #define GET_SUBTARGETINFO_HEADER
#include "R600GenSubtargetInfo.inc" #include "R600GenSubtargetInfo.inc"
namespace llvm { namespace llvm {
class StringRef; class StringRef;
class AMDGPUCommonSubtarget { class AMDGPUSubtarget {
private: private:
Triple TargetTriple; Triple TargetTriple;
protected: protected:
const FeatureBitset &SubtargetFeatureBits; const FeatureBitset &SubtargetFeatureBits;
bool Has16BitInsts; bool Has16BitInsts;
bool HasMadMixInsts; bool HasMadMixInsts;
bool FP32Denormals; bool FP32Denormals;
bool FPExceptions; bool FPExceptions;
bool HasSDWA; bool HasSDWA;
bool HasVOP3PInsts; bool HasVOP3PInsts;
bool HasMulI24; bool HasMulI24;
bool HasMulU24; bool HasMulU24;
bool HasFminFmaxLegacy; bool HasFminFmaxLegacy;
bool EnablePromoteAlloca; bool EnablePromoteAlloca;
int LocalMemorySize; int LocalMemorySize;
unsigned WavefrontSize; unsigned WavefrontSize;
public: public:
AMDGPUCommonSubtarget(const Triple &TT, const FeatureBitset &FeatureBits); AMDGPUSubtarget(const Triple &TT, const FeatureBitset &FeatureBits);
static const AMDGPUCommonSubtarget &get(const MachineFunction &MF); static const AMDGPUSubtarget &get(const MachineFunction &MF);
static const AMDGPUCommonSubtarget &get(const TargetMachine &TM, static const AMDGPUSubtarget &get(const TargetMachine &TM,
const Function &F); const Function &F);
/// \returns Default range flat work group size for a calling convention. /// \returns Default range flat work group size for a calling convention.
std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const; std::pair<unsigned, unsigned> getDefaultFlatWorkGroupSize(CallingConv::ID CC) const;
/// \returns Subtarget's default pair of minimum/maximum flat work group sizes /// \returns Subtarget's default pair of minimum/maximum flat work group sizes
/// for function \p F, or minimum/maximum flat work group sizes explicitly /// for function \p F, or minimum/maximum flat work group sizes explicitly
/// requested using "amdgpu-flat-work-group-size" attribute attached to /// requested using "amdgpu-flat-work-group-size" attribute attached to
▲ Show 20 Lines • Show All 133 Lines • ▼ Show 20 Lines unsigned getMinWavesPerEU() const {
return AMDGPU::IsaInfo::getMinWavesPerEU(SubtargetFeatureBits); return AMDGPU::IsaInfo::getMinWavesPerEU(SubtargetFeatureBits);
} }
unsigned getMaxWavesPerEU() const { return 10; } unsigned getMaxWavesPerEU() const { return 10; }
/// Creates value range metadata on an workitemid.* inrinsic call or load. /// Creates value range metadata on an workitemid.* inrinsic call or load.
bool makeLIDRangeMetadata(Instruction *I) const; bool makeLIDRangeMetadata(Instruction *I) const;
virtual ~AMDGPUCommonSubtarget() {} virtual ~AMDGPUSubtarget() {}
}; };
class AMDGPUSubtarget : public AMDGPUGenSubtargetInfo, class GCNSubtarget : public AMDGPUGenSubtargetInfo,
public AMDGPUCommonSubtarget { public AMDGPUSubtarget {
public: public:
enum Generation { enum Generation {
// Gap for R600 generations, so we can do comparisons between // Gap for R600 generations, so we can do comparisons between
// AMDGPUSubtarget and r600Subtarget. // GCNSubtarget and r600Subtarget.
jveselyUnsubmitted
Done
ReplyInline Actions

Just out of curiosity what's the benefit of having this enum split between GCNSUbtarget and R600Subtarget, vs merging it in AMDGPUSubtarget?

jvesely: Just out of curiosity what's the benefit of having this enum split between GCNSUbtarget and…
tstellarAuthorUnsubmitted
Not Done
ReplyInline Actions

There is no benefit, so I've merged the enums back together.

tstellar: There is no benefit, so I've merged the enums back together.
SOUTHERN_ISLANDS = 4, SOUTHERN_ISLANDS = 4,
SEA_ISLANDS = 5, SEA_ISLANDS = 5,
VOLCANIC_ISLANDS = 6, VOLCANIC_ISLANDS = 6,
GFX9 = 7, GFX9 = 7,
}; };
enum { enum {
ISAVersion0_0_0, ISAVersion0_0_0,
Show All 30 Lines enum TrapID {
TrapIDDebugReservedFF = 0xff TrapIDDebugReservedFF = 0xff
}; };
enum TrapRegValues { enum TrapRegValues {
LLVMTrapHandlerRegValue = 1 LLVMTrapHandlerRegValue = 1
}; };
private: private:
SIFrameLowering FrameLowering;
/// GlobalISel related APIs. /// GlobalISel related APIs.
std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo; std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
std::unique_ptr<InstructionSelector> InstSelector; std::unique_ptr<InstructionSelector> InstSelector;
std::unique_ptr<LegalizerInfo> Legalizer; std::unique_ptr<LegalizerInfo> Legalizer;
std::unique_ptr<RegisterBankInfo> RegBankInfo; std::unique_ptr<RegisterBankInfo> RegBankInfo;
protected: protected:
// Basic subtarget description. // Basic subtarget description.
▲ Show 20 Lines • Show All 68 Lines • ▼ Show 20 Linesprotected:
short TexVTXClauseSize; short TexVTXClauseSize;
bool ScalarizeGlobal; bool ScalarizeGlobal;
// Dummy feature to use for assembler in tablegen. // Dummy feature to use for assembler in tablegen.
bool FeatureDisable; bool FeatureDisable;
SelectionDAGTargetInfo TSInfo; SelectionDAGTargetInfo TSInfo;
AMDGPUAS AS; AMDGPUAS AS;
private:
SITargetLowering TLInfo;
SIInstrInfo InstrInfo;
SIFrameLowering FrameLowering;
public: public:
AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS, GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
const TargetMachine &TM); const GCNTargetMachine &TM);
~AMDGPUSubtarget() override; ~GCNSubtarget() override;
AMDGPUSubtarget &initializeSubtargetDependencies(const Triple &TT, GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,
StringRef GPU, StringRef FS); StringRef GPU, StringRef FS);
virtual const SIInstrInfo *getInstrInfo() const override = 0; const SIInstrInfo *getInstrInfo() const override {
return &InstrInfo;
}
const SIFrameLowering *getFrameLowering() const override { const SIFrameLowering *getFrameLowering() const override {
return &FrameLowering; return &FrameLowering;
} }
virtual const SITargetLowering *getTargetLowering() const override = 0; const SITargetLowering *getTargetLowering() const override {
return &TLInfo;
}
virtual const SIRegisterInfo *getRegisterInfo() const override = 0; const SIRegisterInfo *getRegisterInfo() const override {
return &InstrInfo.getRegisterInfo();
}
const CallLowering *getCallLowering() const override { const CallLowering *getCallLowering() const override {
return CallLoweringInfo.get(); return CallLoweringInfo.get();
} }
const InstructionSelector *getInstructionSelector() const override { const InstructionSelector *getInstructionSelector() const override {
return InstSelector.get(); return InstSelector.get();
} }
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines bool hasFastFMAF32() const {
return FastFMAF32; return FastFMAF32;
} }
bool hasHalfRate64Ops() const { bool hasHalfRate64Ops() const {
return HalfRate64Ops; return HalfRate64Ops;
} }
bool hasAddr64() const { bool hasAddr64() const {
return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS); return (getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS);
} }
bool hasBFE() const { bool hasBFE() const {
return true; return true;
} }
bool hasBFI() const { bool hasBFI() const {
return true; return true;
Show All 11 Lines bool hasFFBL() const {
return true; return true;
} }
bool hasFFBH() const { bool hasFFBH() const {
return true; return true;
} }
bool hasMed3_16() const { bool hasMed3_16() const {
return getGeneration() >= AMDGPUSubtarget::GFX9; return getGeneration() >= GCNSubtarget::GFX9;
} }
bool hasMin3Max3_16() const { bool hasMin3Max3_16() const {
return getGeneration() >= AMDGPUSubtarget::GFX9; return getGeneration() >= GCNSubtarget::GFX9;
} }
bool hasFmaMixInsts() const { bool hasFmaMixInsts() const {
return HasFmaMixInsts; return HasFmaMixInsts;
} }
bool hasCARRY() const { bool hasCARRY() const {
return true; return true;
Show All 28 Lines bool hasFP16Denormals() const {
return FP64FP16Denormals; return FP64FP16Denormals;
} }
bool hasFP64Denormals() const { bool hasFP64Denormals() const {
return FP64FP16Denormals; return FP64FP16Denormals;
} }
bool supportsMinMaxDenormModes() const { bool supportsMinMaxDenormModes() const {
return getGeneration() >= AMDGPUSubtarget::GFX9; return getGeneration() >= GCNSubtarget::GFX9;
} }
bool enableDX10Clamp() const { bool enableDX10Clamp() const {
return DX10Clamp; return DX10Clamp;
} }
bool enableIEEEBit(const MachineFunction &MF) const { bool enableIEEEBit(const MachineFunction &MF) const {
return AMDGPU::isCompute(MF.getFunction().getCallingConv()); return AMDGPU::isCompute(MF.getFunction().getCallingConv());
} }
bool useFlatForGlobal() const { bool useFlatForGlobal() const {
return FlatForGlobal; return FlatForGlobal;
} }
/// \returns If target supports ds_read/write_b128 and user enables generation /// \returns If target supports ds_read/write_b128 and user enables generation
/// of ds_read/write_b128. /// of ds_read/write_b128.
bool useDS128() const { bool useDS128() const {
return CIInsts && EnableDS128; return CIInsts && EnableDS128;
} }
/// \returns If MUBUF instructions always perform range checking, even for /// \returns If MUBUF instructions always perform range checking, even for
/// buffer resources used for private memory access. /// buffer resources used for private memory access.
bool privateMemoryResourceIsRangeChecked() const { bool privateMemoryResourceIsRangeChecked() const {
return getGeneration() < AMDGPUSubtarget::GFX9; return getGeneration() < GCNSubtarget::GFX9;
} }
bool hasAutoWaitcntBeforeBarrier() const { bool hasAutoWaitcntBeforeBarrier() const {
return AutoWaitcntBeforeBarrier; return AutoWaitcntBeforeBarrier;
} }
bool hasCodeObjectV3() const { bool hasCodeObjectV3() const {
return CodeObjectV3; return CodeObjectV3;
▲ Show 20 Lines • Show All 155 Lines • ▼ Show 20 Linespublic:
} }
/// \returns Number of waves per work group supported by the subtarget and /// \returns Number of waves per work group supported by the subtarget and
/// limited by given \p FlatWorkGroupSize. /// limited by given \p FlatWorkGroupSize.
unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const { unsigned getWavesPerWorkGroup(unsigned FlatWorkGroupSize) const {
return AMDGPU::IsaInfo::getWavesPerWorkGroup( return AMDGPU::IsaInfo::getWavesPerWorkGroup(
MCSubtargetInfo::getFeatureBits(), FlatWorkGroupSize); MCSubtargetInfo::getFeatureBits(), FlatWorkGroupSize);
} }
};
class SISubtarget final : public AMDGPUSubtarget {
private:
SIInstrInfo InstrInfo;
SIFrameLowering FrameLowering;
SITargetLowering TLInfo;
/// GlobalISel related APIs.
std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;
std::unique_ptr<InstructionSelector> InstSelector;
std::unique_ptr<LegalizerInfo> Legalizer;
std::unique_ptr<RegisterBankInfo> RegBankInfo;
public:
SISubtarget(const Triple &TT, StringRef CPU, StringRef FS,
const GCNTargetMachine &TM);
const SIInstrInfo *getInstrInfo() const override {
return &InstrInfo;
}
const SIFrameLowering *getFrameLowering() const override {
return &FrameLowering;
}
const SITargetLowering *getTargetLowering() const override {
return &TLInfo;
}
const CallLowering *getCallLowering() const override {
return CallLoweringInfo.get();
}
const InstructionSelector *getInstructionSelector() const override {
return InstSelector.get();
}
const LegalizerInfo *getLegalizerInfo() const override {
return Legalizer.get();
}
const RegisterBankInfo *getRegBankInfo() const override {
return RegBankInfo.get();
}
const SIRegisterInfo *getRegisterInfo() const override {
return &InstrInfo.getRegisterInfo();
}
// static wrappers // static wrappers
static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI); static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);
// XXX - Why is this here if it isn't in the default pass set? // XXX - Why is this here if it isn't in the default pass set?
bool enableEarlyIfConversion() const override { bool enableEarlyIfConversion() const override {
return true; return true;
} }
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines bool loadStoreOptEnabled() const {
return EnableLoadStoreOpt; return EnableLoadStoreOpt;
} }
bool hasSGPRInitBug() const { bool hasSGPRInitBug() const {
return SGPRInitBug; return SGPRInitBug;
} }
bool has12DWordStoreHazard() const { bool has12DWordStoreHazard() const {
return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS; return getGeneration() != GCNSubtarget::SOUTHERN_ISLANDS;
} }
bool hasSMovFedHazard() const { bool hasSMovFedHazard() const {
return getGeneration() >= AMDGPUSubtarget::GFX9; return getGeneration() >= GCNSubtarget::GFX9;
} }
bool hasReadM0MovRelInterpHazard() const { bool hasReadM0MovRelInterpHazard() const {
return getGeneration() >= AMDGPUSubtarget::GFX9; return getGeneration() >= GCNSubtarget::GFX9;
} }
bool hasReadM0SendMsgHazard() const { bool hasReadM0SendMsgHazard() const {
return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS; return getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS;
} }
uint64_t getExplicitKernArgSize(const Function &F) const; uint64_t getExplicitKernArgSize(const Function &F) const;
unsigned getKernArgSegmentSize(const Function &F, unsigned getKernArgSegmentSize(const Function &F,
int64_t ExplicitArgBytes = -1) const; int64_t ExplicitArgBytes = -1) const;
/// Return the maximum number of waves per SIMD for kernels using \p SGPRs /// Return the maximum number of waves per SIMD for kernels using \p SGPRs
/// SGPRs /// SGPRs
unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const; unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;
/// Return the maximum number of waves per SIMD for kernels using \p VGPRs /// Return the maximum number of waves per SIMD for kernels using \p VGPRs
/// VGPRs /// VGPRs
unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const; unsigned getOccupancyWithNumVGPRs(unsigned VGPRs) const;
/// \returns true if the flat_scratch register should be initialized with the /// \returns true if the flat_scratch register should be initialized with the
/// pointer to the wave's scratch memory rather than a size and offset. /// pointer to the wave's scratch memory rather than a size and offset.
bool flatScratchIsPointer() const { bool flatScratchIsPointer() const {
return getGeneration() >= AMDGPUSubtarget::GFX9; return getGeneration() >= GCNSubtarget::GFX9;
} }
/// \returns true if the machine has merged shaders in which s0-s7 are /// \returns true if the machine has merged shaders in which s0-s7 are
/// reserved by the hardware and user SGPRs start at s8 /// reserved by the hardware and user SGPRs start at s8
bool hasMergedShaders() const { bool hasMergedShaders() const {
return getGeneration() >= GFX9; return getGeneration() >= GFX9;
} }
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Linespublic:
/// unit requirement. /// unit requirement.
unsigned getMaxNumVGPRs(const MachineFunction &MF) const; unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
void getPostRAMutations( void getPostRAMutations(
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations)
const override; const override;
}; };
#if 0
class GCNSubtarget final : public GCNSubtarget {
private:
public:
GCNSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
const GCNTargetMachine &TM);
const SIFrameLowering *getFrameLowering() const override {
return &FrameLowering;
}
const CallLowering *getCallLowering() const override {
return CallLoweringInfo.get();
}
const InstructionSelector *getInstructionSelector() const override {
return InstSelector.get();
}
const LegalizerInfo *getLegalizerInfo() const override {
return Legalizer.get();
}
const RegisterBankInfo *getRegBankInfo() const override {
return RegBankInfo.get();
}
};
#endif
jveselyUnsubmitted
Done
ReplyInline Actions

Is this #if 0 code here intentionally? If yes an explanatory comment would be nice.

jvesely: Is this #if 0 code here intentionally? If yes an explanatory comment would be nice.
class R600Subtarget final : public R600GenSubtargetInfo, class R600Subtarget final : public R600GenSubtargetInfo,
public AMDGPUCommonSubtarget { public AMDGPUSubtarget {
public: public:
enum Generation { R600 = 0, R700 = 1, EVERGREEN = 2, NORTHERN_ISLANDS = 3 }; enum Generation { R600 = 0, R700 = 1, EVERGREEN = 2, NORTHERN_ISLANDS = 3 };
private: private:
R600InstrInfo InstrInfo; R600InstrInfo InstrInfo;
R600FrameLowering FrameLowering; R600FrameLowering FrameLowering;
bool FMA; bool FMA;
bool CaymanISA; bool CaymanISA;
▲ Show 20 Lines • Show All 109 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUSubtarget.cpp

Show All 28 Lines
#include <algorithm> #include <algorithm>
using namespace llvm; using namespace llvm;
#define DEBUG_TYPE "amdgpu-subtarget" #define DEBUG_TYPE "amdgpu-subtarget"
#define GET_SUBTARGETINFO_TARGET_DESC #define GET_SUBTARGETINFO_TARGET_DESC
#define GET_SUBTARGETINFO_CTOR #define GET_SUBTARGETINFO_CTOR
#define AMDGPUSubtarget GCNSubtarget
#include "AMDGPUGenSubtargetInfo.inc" #include "AMDGPUGenSubtargetInfo.inc"
#define GET_SUBTARGETINFO_TARGET_DESC #define GET_SUBTARGETINFO_TARGET_DESC
#define GET_SUBTARGETINFO_CTOR #define GET_SUBTARGETINFO_CTOR
#undef AMDGPUSubtarget
#include "R600GenSubtargetInfo.inc" #include "R600GenSubtargetInfo.inc"
AMDGPUSubtarget::~AMDGPUSubtarget() = default; GCNSubtarget::~GCNSubtarget() = default;
R600Subtarget & R600Subtarget &
R600Subtarget::initializeSubtargetDependencies(const Triple &TT, R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
StringRef GPU, StringRef FS) { StringRef GPU, StringRef FS) {
SmallString<256> FullFS("+promote-alloca,+dx10-clamp,"); SmallString<256> FullFS("+promote-alloca,+dx10-clamp,");
FullFS += FS; FullFS += FS;
ParseSubtargetFeatures(GPU, FullFS); ParseSubtargetFeatures(GPU, FullFS);
// FIXME: I don't think think Evergreen has any useful support for // FIXME: I don't think think Evergreen has any useful support for
// denormals, but should be checked. Should we issue a warning somewhere // denormals, but should be checked. Should we issue a warning somewhere
// if someone tries to enable these? // if someone tries to enable these?
if (getGeneration() <= R600Subtarget::NORTHERN_ISLANDS) { if (getGeneration() <= R600Subtarget::NORTHERN_ISLANDS) {
FP32Denormals = false; FP32Denormals = false;
} }
HasMulU24 = getGeneration() >= EVERGREEN; HasMulU24 = getGeneration() >= EVERGREEN;
HasMulI24 = hasCaymanISA(); HasMulI24 = hasCaymanISA();
return *this; return *this;
} }
AMDGPUSubtarget & GCNSubtarget &
AMDGPUSubtarget::initializeSubtargetDependencies(const Triple &TT, GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
StringRef GPU, StringRef FS) { StringRef GPU, StringRef FS) {
// Determine default and user-specified characteristics // Determine default and user-specified characteristics
// On SI+, we want FP64 denormals to be on by default. FP32 denormals can be // On SI+, we want FP64 denormals to be on by default. FP32 denormals can be
// enabled, but some instructions do not respect them and they run at the // enabled, but some instructions do not respect them and they run at the
// double precision rate, so don't enable by default. // double precision rate, so don't enable by default.
// //
// We want to be able to turn these off, but making this a subtarget feature // We want to be able to turn these off, but making this a subtarget feature
// for SI has the unhelpful behavior that it unsets everything else if you // for SI has the unhelpful behavior that it unsets everything else if you
// disable it. // disable it.
SmallString<256> FullFS("+promote-alloca,+dx10-clamp,+load-store-opt,"); SmallString<256> FullFS("+promote-alloca,+dx10-clamp,+load-store-opt,");
if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA. if (isAmdHsaOS()) // Turn on FlatForGlobal for HSA.
FullFS += "+flat-address-space,+flat-for-global,+unaligned-buffer-access,+trap-handler,"; FullFS += "+flat-address-space,+flat-for-global,+unaligned-buffer-access,+trap-handler,";
// FIXME: I don't think think Evergreen has any useful support for // FIXME: I don't think think Evergreen has any useful support for
// denormals, but should be checked. Should we issue a warning somewhere // denormals, but should be checked. Should we issue a warning somewhere
// if someone tries to enable these? // if someone tries to enable these?
if (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS) { if (getGeneration() >= GCNSubtarget::SOUTHERN_ISLANDS) {
FullFS += "+fp64-fp16-denormals,"; FullFS += "+fp64-fp16-denormals,";
} else { } else {
FullFS += "-fp32-denormals,"; FullFS += "-fp32-denormals,";
} }
FullFS += FS; FullFS += FS;
ParseSubtargetFeatures(GPU, FullFS); ParseSubtargetFeatures(GPU, FullFS);
// We don't support FP64 for EG/NI atm. // We don't support FP64 for EG/NI atm.
assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS)); assert(!hasFP64() || (getGeneration() >= GCNSubtarget::SOUTHERN_ISLANDS));
// Unless +-flat-for-global is specified, turn on FlatForGlobal for all OS-es // Unless +-flat-for-global is specified, turn on FlatForGlobal for all OS-es
// on VI and newer hardware to avoid assertion failures due to missing ADDR64 // on VI and newer hardware to avoid assertion failures due to missing ADDR64
// variants of MUBUF instructions. // variants of MUBUF instructions.
if (!hasAddr64() && !FS.contains("flat-for-global")) { if (!hasAddr64() && !FS.contains("flat-for-global")) {
FlatForGlobal = true; FlatForGlobal = true;
} }
// Set defaults if needed. // Set defaults if needed.
if (MaxPrivateElementSize == 0) if (MaxPrivateElementSize == 0)
MaxPrivateElementSize = 4; MaxPrivateElementSize = 4;
if (LDSBankCount == 0) if (LDSBankCount == 0)
LDSBankCount = 32; LDSBankCount = 32;
if (TT.getArch() == Triple::amdgcn) { if (TT.getArch() == Triple::amdgcn) {
if (LocalMemorySize == 0) if (LocalMemorySize == 0)
LocalMemorySize = 32768; LocalMemorySize = 32768;
// Do something sensible for unspecified target. // Do something sensible for unspecified target.
if (!HasMovrel && !HasVGPRIndexMode) if (!HasMovrel && !HasVGPRIndexMode)
HasMovrel = true; HasMovrel = true;
} }
HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS; HasFminFmaxLegacy = getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS;
return *this; return *this;
} }
AMDGPUCommonSubtarget::AMDGPUCommonSubtarget(const Triple &TT, AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT,
const FeatureBitset &FeatureBits) : const FeatureBitset &FeatureBits) :
TargetTriple(TT), TargetTriple(TT),
SubtargetFeatureBits(FeatureBits), SubtargetFeatureBits(FeatureBits),
Has16BitInsts(false), Has16BitInsts(false),
HasMadMixInsts(false), HasMadMixInsts(false),
FP32Denormals(false), FP32Denormals(false),
FPExceptions(false), FPExceptions(false),
HasSDWA(false), HasSDWA(false),
HasVOP3PInsts(false), HasVOP3PInsts(false),
HasMulI24(true), HasMulI24(true),
HasMulU24(true), HasMulU24(true),
HasFminFmaxLegacy(true), HasFminFmaxLegacy(true),
EnablePromoteAlloca(false), EnablePromoteAlloca(false),
LocalMemorySize(0), LocalMemorySize(0),
WavefrontSize(0) WavefrontSize(0)
{ } { }
AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT, StringRef GPU, StringRef FS, GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
const TargetMachine &TM) : const GCNTargetMachine &TM) :
AMDGPUGenSubtargetInfo(TT, GPU, FS), AMDGPUGenSubtargetInfo(TT, GPU, FS),
AMDGPUCommonSubtarget(TT, getFeatureBits()), AMDGPUSubtarget(TT, getFeatureBits()),
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
TargetTriple(TT), TargetTriple(TT),
Gen(SOUTHERN_ISLANDS), Gen(SOUTHERN_ISLANDS),
IsaVersion(ISAVersion0_0_0), IsaVersion(ISAVersion0_0_0),
LDSBankCount(0), LDSBankCount(0),
MaxPrivateElementSize(0), MaxPrivateElementSize(0),
FastFMAF32(false), FastFMAF32(false),
HalfRate64Ops(false), HalfRate64Ops(false),
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 LinesGCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
FlatInstOffsets(false), FlatInstOffsets(false),
FlatGlobalInsts(false), FlatGlobalInsts(false),
FlatScratchInsts(false), FlatScratchInsts(false),
AddNoCarryInsts(false), AddNoCarryInsts(false),
HasUnpackedD16VMem(false), HasUnpackedD16VMem(false),
ScalarizeGlobal(false), ScalarizeGlobal(false),
FeatureDisable(false) { FeatureDisable(false),
TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)),
InstrInfo(*this),
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0) {
CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo()));
InstSelector.reset(new AMDGPUInstructionSelector(
*this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
AS = AMDGPU::getAMDGPUAS(TT); AS = AMDGPU::getAMDGPUAS(TT);
initializeSubtargetDependencies(TT, GPU, FS); initializeSubtargetDependencies(TT, GPU, FS);
} }
unsigned AMDGPUCommonSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves, unsigned AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
const Function &F) const { const Function &F) const {
if (NWaves == 1) if (NWaves == 1)
return getLocalMemorySize(); return getLocalMemorySize();
unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second; unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize); unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
unsigned MaxWaves = getMaxWavesPerEU(); unsigned MaxWaves = getMaxWavesPerEU();
return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves; return getLocalMemorySize() * MaxWaves / WorkGroupsPerCu / NWaves;
} }
unsigned AMDGPUCommonSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes, unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
const Function &F) const { const Function &F) const {
unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second; unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize); unsigned WorkGroupsPerCu = getMaxWorkGroupsPerCU(WorkGroupSize);
unsigned MaxWaves = getMaxWavesPerEU(); unsigned MaxWaves = getMaxWavesPerEU();
unsigned Limit = getLocalMemorySize() * MaxWaves / WorkGroupsPerCu; unsigned Limit = getLocalMemorySize() * MaxWaves / WorkGroupsPerCu;
unsigned NumWaves = Limit / (Bytes ? Bytes : 1u); unsigned NumWaves = Limit / (Bytes ? Bytes : 1u);
NumWaves = std::min(NumWaves, MaxWaves); NumWaves = std::min(NumWaves, MaxWaves);
NumWaves = std::max(NumWaves, 1u); NumWaves = std::max(NumWaves, 1u);
return NumWaves; return NumWaves;
} }
unsigned unsigned
AMDGPUCommonSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const { AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const {
const auto *MFI = MF.getInfo<SIMachineFunctionInfo>(); const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction()); return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction());
} }
std::pair<unsigned, unsigned> std::pair<unsigned, unsigned>
AMDGPUCommonSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const { AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
switch (CC) { switch (CC) {
case CallingConv::AMDGPU_CS: case CallingConv::AMDGPU_CS:
case CallingConv::AMDGPU_KERNEL: case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL: case CallingConv::SPIR_KERNEL:
return std::make_pair(getWavefrontSize() * 2, getWavefrontSize() * 4); return std::make_pair(getWavefrontSize() * 2, getWavefrontSize() * 4);
case CallingConv::AMDGPU_VS: case CallingConv::AMDGPU_VS:
case CallingConv::AMDGPU_LS: case CallingConv::AMDGPU_LS:
case CallingConv::AMDGPU_HS: case CallingConv::AMDGPU_HS:
case CallingConv::AMDGPU_ES: case CallingConv::AMDGPU_ES:
case CallingConv::AMDGPU_GS: case CallingConv::AMDGPU_GS:
case CallingConv::AMDGPU_PS: case CallingConv::AMDGPU_PS:
return std::make_pair(1, getWavefrontSize()); return std::make_pair(1, getWavefrontSize());
default: default:
return std::make_pair(1, 16 * getWavefrontSize()); return std::make_pair(1, 16 * getWavefrontSize());
} }
} }
std::pair<unsigned, unsigned> AMDGPUCommonSubtarget::getFlatWorkGroupSizes( std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
const Function &F) const { const Function &F) const {
// FIXME: 1024 if function. // FIXME: 1024 if function.
// Default minimum/maximum flat work group sizes. // Default minimum/maximum flat work group sizes.
std::pair<unsigned, unsigned> Default = std::pair<unsigned, unsigned> Default =
getDefaultFlatWorkGroupSize(F.getCallingConv()); getDefaultFlatWorkGroupSize(F.getCallingConv());
// TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa // TODO: Do not process "amdgpu-max-work-group-size" attribute once mesa
// starts using "amdgpu-flat-work-group-size" attribute. // starts using "amdgpu-flat-work-group-size" attribute.
Show All 13 Linesstd::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
if (Requested.first < getMinFlatWorkGroupSize()) if (Requested.first < getMinFlatWorkGroupSize())
return Default; return Default;
if (Requested.second > getMaxFlatWorkGroupSize()) if (Requested.second > getMaxFlatWorkGroupSize())
return Default; return Default;
return Requested; return Requested;
} }
std::pair<unsigned, unsigned> AMDGPUCommonSubtarget::getWavesPerEU( std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
const Function &F) const { const Function &F) const {
// Default minimum/maximum number of waves per execution unit. // Default minimum/maximum number of waves per execution unit.
std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU()); std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
// Default/requested minimum/maximum flat work group sizes. // Default/requested minimum/maximum flat work group sizes.
std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F); std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
// If minimum/maximum flat work group sizes were explicitly requested using // If minimum/maximum flat work group sizes were explicitly requested using
Show All 31 Linesstd::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
// minimum/maximum flat work group sizes. // minimum/maximum flat work group sizes.
if (RequestedFlatWorkGroupSize && if (RequestedFlatWorkGroupSize &&
Requested.first < MinImpliedByFlatWorkGroupSize) Requested.first < MinImpliedByFlatWorkGroupSize)
return Default; return Default;
return Requested; return Requested;
} }
bool AMDGPUCommonSubtarget::makeLIDRangeMetadata(Instruction *I) const { bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
Function *Kernel = I->getParent()->getParent(); Function *Kernel = I->getParent()->getParent();
unsigned MinSize = 0; unsigned MinSize = 0;
unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second; unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
bool IdQuery = false; bool IdQuery = false;
// If reqd_work_group_size is present it narrows value down. // If reqd_work_group_size is present it narrows value down.
if (auto *CI = dyn_cast<CallInst>(I)) { if (auto *CI = dyn_cast<CallInst>(I)) {
const Function *F = CI->getCalledFunction(); const Function *F = CI->getCalledFunction();
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
APInt(32, MaxSize)); APInt(32, MaxSize));
I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange); I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
return true; return true;
} }
R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS, R600Subtarget::R600Subtarget(const Triple &TT, StringRef GPU, StringRef FS,
const TargetMachine &TM) : const TargetMachine &TM) :
R600GenSubtargetInfo(TT, GPU, FS), R600GenSubtargetInfo(TT, GPU, FS),
AMDGPUCommonSubtarget(TT, getFeatureBits()), AMDGPUSubtarget(TT, getFeatureBits()),
InstrInfo(*this), InstrInfo(*this),
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0), FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
FMA(false), FMA(false),
CaymanISA(false), CaymanISA(false),
CFALUBug(false), CFALUBug(false),
DX10Clamp(false), DX10Clamp(false),
HasVertexCache(false), HasVertexCache(false),
R600ALUInst(false), R600ALUInst(false),
FP64(false), FP64(false),
TexVTXClauseSize(0), TexVTXClauseSize(0),
Gen(R600), Gen(R600),
TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)), TLInfo(TM, initializeSubtargetDependencies(TT, GPU, FS)),
InstrItins(getInstrItineraryForCPU(GPU)), InstrItins(getInstrItineraryForCPU(GPU)),
AS (AMDGPU::getAMDGPUAS(TT)) { } AS (AMDGPU::getAMDGPUAS(TT)) { }
SISubtarget::SISubtarget(const Triple &TT, StringRef GPU, StringRef FS, void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
const GCNTargetMachine &TM)
: AMDGPUSubtarget(TT, GPU, FS, TM), InstrInfo(*this),
FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0),
TLInfo(TM, *this) {
CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo()));
InstSelector.reset(new AMDGPUInstructionSelector(
*this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
}
void SISubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
unsigned NumRegionInstrs) const { unsigned NumRegionInstrs) const {
// Track register pressure so the scheduler can try to decrease // Track register pressure so the scheduler can try to decrease
// pressure once register usage is above the threshold defined by // pressure once register usage is above the threshold defined by
// SIRegisterInfo::getRegPressureSetLimit() // SIRegisterInfo::getRegPressureSetLimit()
Policy.ShouldTrackPressure = true; Policy.ShouldTrackPressure = true;
// Enabling both top down and bottom up scheduling seems to give us less // Enabling both top down and bottom up scheduling seems to give us less
// register spills than just using one of these approaches on its own. // register spills than just using one of these approaches on its own.
Policy.OnlyTopDown = false; Policy.OnlyTopDown = false;
Policy.OnlyBottomUp = false; Policy.OnlyBottomUp = false;
// Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler. // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
if (!enableSIScheduler()) if (!enableSIScheduler())
Policy.ShouldTrackLaneMasks = true; Policy.ShouldTrackLaneMasks = true;
} }
bool SISubtarget::isVGPRSpillingEnabled(const Function& F) const { bool GCNSubtarget::isVGPRSpillingEnabled(const Function& F) const {
return EnableVGPRSpilling || !AMDGPU::isShader(F.getCallingConv()); return EnableVGPRSpilling || !AMDGPU::isShader(F.getCallingConv());
} }
uint64_t SISubtarget::getExplicitKernArgSize(const Function &F) const { uint64_t GCNSubtarget::getExplicitKernArgSize(const Function &F) const {
assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL); assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL);
const DataLayout &DL = F.getParent()->getDataLayout(); const DataLayout &DL = F.getParent()->getDataLayout();
uint64_t ExplicitArgBytes = 0; uint64_t ExplicitArgBytes = 0;
for (const Argument &Arg : F.args()) { for (const Argument &Arg : F.args()) {
Type *ArgTy = Arg.getType(); Type *ArgTy = Arg.getType();
unsigned Align = DL.getABITypeAlignment(ArgTy); unsigned Align = DL.getABITypeAlignment(ArgTy);
uint64_t AllocSize = DL.getTypeAllocSize(ArgTy); uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
ExplicitArgBytes = alignTo(ExplicitArgBytes, Align) + AllocSize; ExplicitArgBytes = alignTo(ExplicitArgBytes, Align) + AllocSize;
} }
return ExplicitArgBytes; return ExplicitArgBytes;
} }
unsigned SISubtarget::getKernArgSegmentSize(const Function &F, unsigned GCNSubtarget::getKernArgSegmentSize(const Function &F,
int64_t ExplicitArgBytes) const { int64_t ExplicitArgBytes) const {
if (ExplicitArgBytes == -1) if (ExplicitArgBytes == -1)
ExplicitArgBytes = getExplicitKernArgSize(F); ExplicitArgBytes = getExplicitKernArgSize(F);
unsigned ExplicitOffset = getExplicitKernelArgOffset(F); unsigned ExplicitOffset = getExplicitKernelArgOffset(F);
uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes; uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
unsigned ImplicitBytes = getImplicitArgNumBytes(F); unsigned ImplicitBytes = getImplicitArgNumBytes(F);
if (ImplicitBytes != 0) { if (ImplicitBytes != 0) {
unsigned Alignment = getAlignmentForImplicitArgPtr(); unsigned Alignment = getAlignmentForImplicitArgPtr();
TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes; TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
} }
// Being able to dereference past the end is useful for emitting scalar loads. // Being able to dereference past the end is useful for emitting scalar loads.
return alignTo(TotalSize, 4); return alignTo(TotalSize, 4);
} }
unsigned SISubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const { unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
if (getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
if (SGPRs <= 80) if (SGPRs <= 80)
return 10; return 10;
if (SGPRs <= 88) if (SGPRs <= 88)
return 9; return 9;
if (SGPRs <= 100) if (SGPRs <= 100)
return 8; return 8;
return 7; return 7;
} }
if (SGPRs <= 48) if (SGPRs <= 48)
return 10; return 10;
if (SGPRs <= 56) if (SGPRs <= 56)
return 9; return 9;
if (SGPRs <= 64) if (SGPRs <= 64)
return 8; return 8;
if (SGPRs <= 72) if (SGPRs <= 72)
return 7; return 7;
if (SGPRs <= 80) if (SGPRs <= 80)
return 6; return 6;
return 5; return 5;
} }
unsigned SISubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const { unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
if (VGPRs <= 24) if (VGPRs <= 24)
return 10; return 10;
if (VGPRs <= 28) if (VGPRs <= 28)
return 9; return 9;
if (VGPRs <= 32) if (VGPRs <= 32)
return 8; return 8;
if (VGPRs <= 36) if (VGPRs <= 36)
return 7; return 7;
if (VGPRs <= 40) if (VGPRs <= 40)
return 6; return 6;
if (VGPRs <= 48) if (VGPRs <= 48)
return 5; return 5;
if (VGPRs <= 64) if (VGPRs <= 64)
return 4; return 4;
if (VGPRs <= 84) if (VGPRs <= 84)
return 3; return 3;
if (VGPRs <= 128) if (VGPRs <= 128)
return 2; return 2;
return 1; return 1;
} }
unsigned SISubtarget::getReservedNumSGPRs(const MachineFunction &MF) const { unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
if (MFI.hasFlatScratchInit()) { if (MFI.hasFlatScratchInit()) {
if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) if (getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS)
return 6; // FLAT_SCRATCH, XNACK, VCC (in that order). return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) if (getGeneration() == GCNSubtarget::SEA_ISLANDS)
return 4; // FLAT_SCRATCH, VCC (in that order). return 4; // FLAT_SCRATCH, VCC (in that order).
} }
if (isXNACKEnabled()) if (isXNACKEnabled())
return 4; // XNACK, VCC (in that order). return 4; // XNACK, VCC (in that order).
return 2; // VCC. return 2; // VCC.
} }
unsigned SISubtarget::getMaxNumSGPRs(const MachineFunction &MF) const { unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
const Function &F = MF.getFunction(); const Function &F = MF.getFunction();
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
// Compute maximum number of SGPRs function can use using default/requested // Compute maximum number of SGPRs function can use using default/requested
// minimum number of waves per execution unit. // minimum number of waves per execution unit.
std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU(); std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false); unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true); unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
Show All 33 Linesunsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
if (hasSGPRInitBug()) if (hasSGPRInitBug())
MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG; MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF), return std::min(MaxNumSGPRs - getReservedNumSGPRs(MF),
MaxAddressableNumSGPRs); MaxAddressableNumSGPRs);
} }
unsigned SISubtarget::getMaxNumVGPRs(const MachineFunction &MF) const { unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
const Function &F = MF.getFunction(); const Function &F = MF.getFunction();
const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
// Compute maximum number of VGPRs function can use using default/requested // Compute maximum number of VGPRs function can use using default/requested
// minimum number of waves per execution unit. // minimum number of waves per execution unit.
std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU(); std::pair<unsigned, unsigned> WavesPerEU = MFI.getWavesPerEU();
unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first); unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines for (SUnit &SU : DAG->SUnits) {
} }
SUa = &SU; SUa = &SU;
} }
} }
}; };
} // namespace } // namespace
void SISubtarget::getPostRAMutations( void GCNSubtarget::getPostRAMutations(
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const { std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
Mutations.push_back(llvm::make_unique<MemOpClusterMutation>(&InstrInfo)); Mutations.push_back(llvm::make_unique<MemOpClusterMutation>(&InstrInfo));
} }
const AMDGPUCommonSubtarget &AMDGPUCommonSubtarget::get(const MachineFunction &MF) { const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn) if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
return static_cast<const AMDGPUCommonSubtarget&>(MF.getSubtarget<AMDGPUSubtarget>()); return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
else else
return static_cast<const AMDGPUCommonSubtarget&>(MF.getSubtarget<R600Subtarget>()); return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<R600Subtarget>());
} }
const AMDGPUCommonSubtarget &AMDGPUCommonSubtarget::get(const TargetMachine &TM, const Function &F) { const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
if (TM.getTargetTriple().getArch() == Triple::amdgcn) if (TM.getTargetTriple().getArch() == Triple::amdgcn)
return static_cast<const AMDGPUCommonSubtarget&>(TM.getSubtarget<AMDGPUSubtarget>(F)); return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
else else
return static_cast<const AMDGPUCommonSubtarget&>(TM.getSubtarget<R600Subtarget>(F)); return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<R600Subtarget>(F));
} }

lib/Target/AMDGPU/AMDGPUTargetMachine.h

Show First 20 Lines • Show All 93 Lines • ▼ Show 20 Lines
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// GCN Target Machine (SI+) // GCN Target Machine (SI+)
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
class GCNTargetMachine final : public AMDGPUTargetMachine { class GCNTargetMachine final : public AMDGPUTargetMachine {
private: private:
AMDGPUIntrinsicInfo IntrinsicInfo; AMDGPUIntrinsicInfo IntrinsicInfo;
mutable StringMap<std::unique_ptr<SISubtarget>> SubtargetMap; mutable StringMap<std::unique_ptr<GCNSubtarget>> SubtargetMap;
public: public:
GCNTargetMachine(const Target &T, const Triple &TT, StringRef CPU, GCNTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
StringRef FS, TargetOptions Options, StringRef FS, TargetOptions Options,
Optional<Reloc::Model> RM, Optional<CodeModel::Model> CM, Optional<Reloc::Model> RM, Optional<CodeModel::Model> CM,
CodeGenOpt::Level OL, bool JIT); CodeGenOpt::Level OL, bool JIT);
TargetPassConfig *createPassConfig(PassManagerBase &PM) override; TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
const SISubtarget *getSubtargetImpl(const Function &) const override; const GCNSubtarget *getSubtargetImpl(const Function &) const override;
TargetTransformInfo getTargetTransformInfo(const Function &F) override; TargetTransformInfo getTargetTransformInfo(const Function &F) override;
const AMDGPUIntrinsicInfo *getIntrinsicInfo() const override { const AMDGPUIntrinsicInfo *getIntrinsicInfo() const override {
return &IntrinsicInfo; return &IntrinsicInfo;
} }
bool useIPRA() const override { bool useIPRA() const override {
return true; return true;
} }
}; };
} // end namespace llvm } // end namespace llvm
#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

Show First 20 Lines • Show All 458 Lines • ▼ Show 20 Lines
GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT, GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
StringRef CPU, StringRef FS, StringRef CPU, StringRef FS,
TargetOptions Options, TargetOptions Options,
Optional<Reloc::Model> RM, Optional<Reloc::Model> RM,
Optional<CodeModel::Model> CM, Optional<CodeModel::Model> CM,
CodeGenOpt::Level OL, bool JIT) CodeGenOpt::Level OL, bool JIT)
: AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {} : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}
const SISubtarget *GCNTargetMachine::getSubtargetImpl(const Function &F) const { const GCNSubtarget *GCNTargetMachine::getSubtargetImpl(const Function &F) const {
StringRef GPU = getGPUName(F); StringRef GPU = getGPUName(F);
StringRef FS = getFeatureString(F); StringRef FS = getFeatureString(F);
SmallString<128> SubtargetKey(GPU); SmallString<128> SubtargetKey(GPU);
SubtargetKey.append(FS); SubtargetKey.append(FS);
auto &I = SubtargetMap[SubtargetKey]; auto &I = SubtargetMap[SubtargetKey];
if (!I) { if (!I) {
// This needs to be done before we create a new subtarget since any // This needs to be done before we create a new subtarget since any
// creation will depend on the TM and the code generation flags on the // creation will depend on the TM and the code generation flags on the
// function that reside in TargetOptions. // function that reside in TargetOptions.
resetTargetOptions(F); resetTargetOptions(F);
I = llvm::make_unique<SISubtarget>(TargetTriple, GPU, FS, *this); I = llvm::make_unique<GCNSubtarget>(TargetTriple, GPU, FS, *this);
} }
I->setScalarizeGlobalBehavior(ScalarizeGlobal); I->setScalarizeGlobalBehavior(ScalarizeGlobal);
return I.get(); return I.get();
} }
TargetTransformInfo TargetTransformInfo
▲ Show 20 Lines • Show All 254 Lines • ▼ Show 20 Lines
} }
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// GCN Pass Setup // GCN Pass Setup
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler( ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
MachineSchedContext *C) const { MachineSchedContext *C) const {
const SISubtarget &ST = C->MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
if (ST.enableSIScheduler()) if (ST.enableSIScheduler())
return createSIMachineScheduler(C); return createSIMachineScheduler(C);
return createGCNMaxOccupancyMachineScheduler(C); return createGCNMaxOccupancyMachineScheduler(C);
} }
bool GCNPassConfig::addPreISel() { bool GCNPassConfig::addPreISel() {
AMDGPUPassConfig::addPreISel(); AMDGPUPassConfig::addPreISel();
▲ Show 20 Lines • Show All 148 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

Show First 20 Lines • Show All 56 Lines • ▼ Show 20 Lines
}; };
class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> { class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
using BaseT = BasicTTIImplBase<GCNTTIImpl>; using BaseT = BasicTTIImplBase<GCNTTIImpl>;
using TTI = TargetTransformInfo; using TTI = TargetTransformInfo;
friend BaseT; friend BaseT;
const AMDGPUSubtarget *ST; const GCNSubtarget *ST;
const AMDGPUTargetLowering *TLI; const AMDGPUTargetLowering *TLI;
AMDGPUTTIImpl CommonTTI; AMDGPUTTIImpl CommonTTI;
bool IsGraphicsShader; bool IsGraphicsShader;
const FeatureBitset InlineFeatureIgnoreList = { const FeatureBitset InlineFeatureIgnoreList = {
// Codegen control options which don't matter. // Codegen control options which don't matter.
AMDGPU::FeatureEnableLoadStoreOpt, AMDGPU::FeatureEnableLoadStoreOpt,
AMDGPU::FeatureEnableSIScheduler, AMDGPU::FeatureEnableSIScheduler,
Show All 12 Lines const FeatureBitset InlineFeatureIgnoreList = {
AMDGPU::FeatureXNACK, AMDGPU::FeatureXNACK,
AMDGPU::FeatureTrapHandler, AMDGPU::FeatureTrapHandler,
// Perf-tuning features // Perf-tuning features
AMDGPU::FeatureFastFMAF32, AMDGPU::FeatureFastFMAF32,
AMDGPU::HalfRate64Ops AMDGPU::HalfRate64Ops
}; };
const AMDGPUSubtarget *getST() const { return ST; } const GCNSubtarget *getST() const { return ST; }
const AMDGPUTargetLowering *getTLI() const { return TLI; } const AMDGPUTargetLowering *getTLI() const { return TLI; }
static inline int getFullRateInstrCost() { static inline int getFullRateInstrCost() {
return TargetTransformInfo::TCC_Basic; return TargetTransformInfo::TCC_Basic;
} }
static inline int getHalfRateInstrCost() { static inline int getHalfRateInstrCost() {
return 2 * TargetTransformInfo::TCC_Basic; return 2 * TargetTransformInfo::TCC_Basic;
Show All 10 Linesclass GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
inline int get64BitInstrCost() const { inline int get64BitInstrCost() const {
return ST->hasHalfRate64Ops() ? return ST->hasHalfRate64Ops() ?
getHalfRateInstrCost() : getQuarterRateInstrCost(); getHalfRateInstrCost() : getQuarterRateInstrCost();
} }
public: public:
explicit GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F) explicit GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
: BaseT(TM, F.getParent()->getDataLayout()), : BaseT(TM, F.getParent()->getDataLayout()),
ST(static_cast<const AMDGPUSubtarget*>(TM->getSubtargetImpl(F))), ST(static_cast<const GCNSubtarget*>(TM->getSubtargetImpl(F))),
TLI(ST->getTargetLowering()), TLI(ST->getTargetLowering()),
CommonTTI(TM, F), CommonTTI(TM, F),
IsGraphicsShader(AMDGPU::isShader(F.getCallingConv())) {} IsGraphicsShader(AMDGPU::isShader(F.getCallingConv())) {}
bool hasBranchDivergence() { return true; } bool hasBranchDivergence() { return true; }
void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
TTI::UnrollingPreferences &UP); TTI::UnrollingPreferences &UP);
▲ Show 20 Lines • Show All 116 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

Show First 20 Lines • Show All 395 Lines • ▼ Show 20 Lines case ISD::FMUL:
break; break;
case ISD::FDIV: case ISD::FDIV:
case ISD::FREM: case ISD::FREM:
// FIXME: frem should be handled separately. The fdiv in it is most of it, // FIXME: frem should be handled separately. The fdiv in it is most of it,
// but the current lowering is also not entirely correct. // but the current lowering is also not entirely correct.
if (SLT == MVT::f64) { if (SLT == MVT::f64) {
int Cost = 4 * get64BitInstrCost() + 7 * getQuarterRateInstrCost(); int Cost = 4 * get64BitInstrCost() + 7 * getQuarterRateInstrCost();
// Add cost of workaround. // Add cost of workaround.
if (ST->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) if (ST->getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS)
Cost += 3 * getFullRateInstrCost(); Cost += 3 * getFullRateInstrCost();
return LT.first * Cost * NElts; return LT.first * Cost * NElts;
} }
if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) { if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) {
// TODO: This is more complicated, unsafe flags etc. // TODO: This is more complicated, unsafe flags etc.
if ((SLT == MVT::f32 && !ST->hasFP32Denormals()) || if ((SLT == MVT::f32 && !ST->hasFP32Denormals()) ||
▲ Show 20 Lines • Show All 320 Lines • Show Last 20 Lines

lib/Target/AMDGPU/GCNHazardRecognizer.h

Show All 22 Lines
class MachineFunction; class MachineFunction;
class MachineInstr; class MachineInstr;
class MachineOperand; class MachineOperand;
class MachineRegisterInfo; class MachineRegisterInfo;
class ScheduleDAG; class ScheduleDAG;
class SIInstrInfo; class SIInstrInfo;
class SIRegisterInfo; class SIRegisterInfo;
class SISubtarget; class GCNSubtarget;
class GCNHazardRecognizer final : public ScheduleHazardRecognizer { class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
// This variable stores the instruction that has been emitted this cycle. It // This variable stores the instruction that has been emitted this cycle. It
// will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
// called. // called.
MachineInstr *CurrCycleInstr; MachineInstr *CurrCycleInstr;
std::list<MachineInstr*> EmittedInstrs; std::list<MachineInstr*> EmittedInstrs;
const MachineFunction &MF; const MachineFunction &MF;
const SISubtarget &ST; const GCNSubtarget &ST;
const SIInstrInfo &TII; const SIInstrInfo &TII;
const SIRegisterInfo &TRI; const SIRegisterInfo &TRI;
/// RegUnits of uses in the current soft memory clause. /// RegUnits of uses in the current soft memory clause.
BitVector ClauseUses; BitVector ClauseUses;
/// RegUnits of defs in the current soft memory clause. /// RegUnits of defs in the current soft memory clause.
BitVector ClauseDefs; BitVector ClauseDefs;
▲ Show 20 Lines • Show All 46 Lines • Show Last 20 Lines

lib/Target/AMDGPU/GCNHazardRecognizer.cpp

Show All 34 Lines
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// Hazard Recoginizer Implementation // Hazard Recoginizer Implementation
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) : GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
CurrCycleInstr(nullptr), CurrCycleInstr(nullptr),
MF(MF), MF(MF),
ST(MF.getSubtarget<SISubtarget>()), ST(MF.getSubtarget<GCNSubtarget>()),
TII(*ST.getInstrInfo()), TII(*ST.getInstrInfo()),
TRI(TII.getRegisterInfo()), TRI(TII.getRegisterInfo()),
ClauseUses(TRI.getNumRegUnits()), ClauseUses(TRI.getNumRegUnits()),
ClauseDefs(TRI.getNumRegUnits()) { ClauseDefs(TRI.getNumRegUnits()) {
MaxLookAhead = 5; MaxLookAhead = 5;
} }
void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { void GCNHazardRecognizer::EmitInstruction(SUnit *SU) {
▲ Show 20 Lines • Show All 299 Lines • ▼ Show 20 Linesint GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) {
addClauseInst(*MEM); addClauseInst(*MEM);
// If the set of defs and uses intersect then we cannot add this instruction // If the set of defs and uses intersect then we cannot add this instruction
// to the clause, so we have a hazard. // to the clause, so we have a hazard.
return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0; return ClauseDefs.anyCommon(ClauseUses) ? 1 : 0;
} }
int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
int WaitStatesNeeded = 0; int WaitStatesNeeded = 0;
WaitStatesNeeded = checkSoftClauseHazards(SMRD); WaitStatesNeeded = checkSoftClauseHazards(SMRD);
// This SMRD hazard only affects SI. // This SMRD hazard only affects SI.
if (ST.getGeneration() != SISubtarget::SOUTHERN_ISLANDS) if (ST.getGeneration() != GCNSubtarget::SOUTHERN_ISLANDS)
return WaitStatesNeeded; return WaitStatesNeeded;
// A read of an SGPR by SMRD instruction requires 4 wait states when the // A read of an SGPR by SMRD instruction requires 4 wait states when the
// SGPR was written by a VALU instruction. // SGPR was written by a VALU instruction.
int SmrdSgprWaitStates = 4; int SmrdSgprWaitStates = 4;
auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); }; auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); }; auto IsBufferHazardDefFn = [this] (MachineInstr *MI) { return TII.isSALU(*MI); };
Show All 20 Lines if (IsBufferSMRD) {
WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse); WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
} }
} }
return WaitStatesNeeded; return WaitStatesNeeded;
} }
int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
if (ST.getGeneration() < SISubtarget::VOLCANIC_ISLANDS) if (ST.getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS)
return 0; return 0;
int WaitStatesNeeded = checkSoftClauseHazards(VMEM); int WaitStatesNeeded = checkSoftClauseHazards(VMEM);
// A read of an SGPR by a VMEM instruction requires 5 wait states when the // A read of an SGPR by a VMEM instruction requires 5 wait states when the
// SGPR was written by a VALU Instruction. // SGPR was written by a VALU Instruction.
const int VmemSgprWaitStates = 5; const int VmemSgprWaitStates = 5;
auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); }; auto IsHazardDefFn = [this] (MachineInstr *MI) { return TII.isVALU(*MI); };
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Linesint GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
return GetRegWaitStates - WaitStatesNeeded; return GetRegWaitStates - WaitStatesNeeded;
} }
int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
unsigned HWReg = getHWReg(TII, *SetRegInstr); unsigned HWReg = getHWReg(TII, *SetRegInstr);
const int SetRegWaitStates = const int SetRegWaitStates =
ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ? 1 : 2; ST.getGeneration() <= GCNSubtarget::SEA_ISLANDS ? 1 : 2;
auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) { auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
return HWReg == getHWReg(TII, *MI); return HWReg == getHWReg(TII, *MI);
}; };
int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn); int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
return SetRegWaitStates - WaitStatesNeeded; return SetRegWaitStates - WaitStatesNeeded;
} }
int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) {
▲ Show 20 Lines • Show All 128 Lines • ▼ Show 20 Linesint GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
}; };
const int RWLaneWaitStates = 4; const int RWLaneWaitStates = 4;
int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn); int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn);
return RWLaneWaitStates - WaitStatesSince; return RWLaneWaitStates - WaitStatesSince;
} }
int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS) if (ST.getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS)
return 0; return 0;
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const int RFEWaitStates = 1; const int RFEWaitStates = 1;
auto IsHazardFn = [TII] (MachineInstr *MI) { auto IsHazardFn = [TII] (MachineInstr *MI) {
return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS; return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
Show All 40 Lines

lib/Target/AMDGPU/GCNIterativeScheduler.cpp

Show First 20 Lines • Show All 102 Lines • ▼ Show 20 Linesstatic void printLivenessInfo(raw_ostream &OS,
const auto BottomMI = End == BB->end() ? std::prev(End) : End; const auto BottomMI = End == BB->end() ? std::prev(End) : End;
const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS); const auto LiveOuts = getLiveRegsAfter(*BottomMI, *LIS);
OS << "LOt RP: "; OS << "LOt RP: ";
getRegPressure(MRI, LiveOuts).print(OS); getRegPressure(MRI, LiveOuts).print(OS);
} }
LLVM_DUMP_METHOD LLVM_DUMP_METHOD
void GCNIterativeScheduler::printRegions(raw_ostream &OS) const { void GCNIterativeScheduler::printRegions(raw_ostream &OS) const {
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
for (const auto R : Regions) { for (const auto R : Regions) {
OS << "Region to schedule "; OS << "Region to schedule ";
printRegion(OS, R->Begin, R->End, LIS, 1); printRegion(OS, R->Begin, R->End, LIS, 1);
printLivenessInfo(OS, R->Begin, R->End, LIS); printLivenessInfo(OS, R->Begin, R->End, LIS);
OS << "Max RP: "; OS << "Max RP: ";
R->MaxPressure.print(OS, &ST); R->MaxPressure.print(OS, &ST);
} }
} }
LLVM_DUMP_METHOD LLVM_DUMP_METHOD
void GCNIterativeScheduler::printSchedResult(raw_ostream &OS, void GCNIterativeScheduler::printSchedResult(raw_ostream &OS,
const Region *R, const Region *R,
const GCNRegPressure &RP) const { const GCNRegPressure &RP) const {
OS << "\nAfter scheduling "; OS << "\nAfter scheduling ";
printRegion(OS, R->Begin, R->End, LIS); printRegion(OS, R->Begin, R->End, LIS);
printSchedRP(OS, R->MaxPressure, RP); printSchedRP(OS, R->MaxPressure, RP);
OS << '\n'; OS << '\n';
} }
LLVM_DUMP_METHOD LLVM_DUMP_METHOD
void GCNIterativeScheduler::printSchedRP(raw_ostream &OS, void GCNIterativeScheduler::printSchedRP(raw_ostream &OS,
const GCNRegPressure &Before, const GCNRegPressure &Before,
const GCNRegPressure &After) const { const GCNRegPressure &After) const {
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
OS << "RP before: "; OS << "RP before: ";
Before.print(OS, &ST); Before.print(OS, &ST);
OS << "RP after: "; OS << "RP after: ";
After.print(OS, &ST); After.print(OS, &ST);
} }
#endif #endif
// DAG builder helper // DAG builder helper
▲ Show 20 Lines • Show All 167 Lines • ▼ Show 20 Lines
} }
void GCNIterativeScheduler::schedule() { // overriden void GCNIterativeScheduler::schedule() { // overriden
// do nothing // do nothing
LLVM_DEBUG(printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS); LLVM_DEBUG(printLivenessInfo(dbgs(), RegionBegin, RegionEnd, LIS);
if (!Regions.empty() && Regions.back()->Begin == RegionBegin) { if (!Regions.empty() && Regions.back()->Begin == RegionBegin) {
dbgs() << "Max RP: "; dbgs() << "Max RP: ";
Regions.back()->MaxPressure.print( Regions.back()->MaxPressure.print(
dbgs(), &MF.getSubtarget<SISubtarget>()); dbgs(), &MF.getSubtarget<GCNSubtarget>());
} dbgs() } dbgs()
<< '\n';); << '\n';);
} }
void GCNIterativeScheduler::finalizeSchedule() { // overriden void GCNIterativeScheduler::finalizeSchedule() { // overriden
if (Regions.empty()) if (Regions.empty())
return; return;
switch (Strategy) { switch (Strategy) {
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines if (!std::is_same<decltype(*Schedule.begin()), MachineInstr*>::value) {
RegionEnd = R.End; RegionEnd = R.End;
} }
R.Begin = RegionBegin; R.Begin = RegionBegin;
R.MaxPressure = MaxRP; R.MaxPressure = MaxRP;
#ifndef NDEBUG #ifndef NDEBUG
const auto RegionMaxRP = getRegionPressure(R); const auto RegionMaxRP = getRegionPressure(R);
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
#endif #endif
assert((SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP)) assert((SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP))
|| (dbgs() << "Max RP mismatch!!!\n" || (dbgs() << "Max RP mismatch!!!\n"
"RP for schedule (calculated): ", "RP for schedule (calculated): ",
SchedMaxRP.print(dbgs(), &ST), SchedMaxRP.print(dbgs(), &ST),
dbgs() << "RP for schedule (reported): ", dbgs() << "RP for schedule (reported): ",
MaxRP.print(dbgs(), &ST), MaxRP.print(dbgs(), &ST),
dbgs() << "RP after scheduling: ", dbgs() << "RP after scheduling: ",
RegionMaxRP.print(dbgs(), &ST), RegionMaxRP.print(dbgs(), &ST),
false)); false));
} }
// Sort recorded regions by pressure - highest at the front // Sort recorded regions by pressure - highest at the front
void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) { void GCNIterativeScheduler::sortRegionsByPressure(unsigned TargetOcc) {
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
llvm::sort(Regions.begin(), Regions.end(), llvm::sort(Regions.begin(), Regions.end(),
[&ST, TargetOcc](const Region *R1, const Region *R2) { [&ST, TargetOcc](const Region *R1, const Region *R2) {
return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc); return R2->MaxPressure.less(ST, R1->MaxPressure, TargetOcc);
}); });
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Legacy MaxOccupancy Strategy // Legacy MaxOccupancy Strategy
// Tries to increase occupancy applying minreg scheduler for a sequence of // Tries to increase occupancy applying minreg scheduler for a sequence of
// most demanding regions. Obtained schedules are saved as BestSchedule for a // most demanding regions. Obtained schedules are saved as BestSchedule for a
// region. // region.
// TargetOcc is the best achievable occupancy for a kernel. // TargetOcc is the best achievable occupancy for a kernel.
// Returns better occupancy on success or current occupancy on fail. // Returns better occupancy on success or current occupancy on fail.
// BestSchedules aren't deleted on fail. // BestSchedules aren't deleted on fail.
unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) { unsigned GCNIterativeScheduler::tryMaximizeOccupancy(unsigned TargetOcc) {
// TODO: assert Regions are sorted descending by pressure // TODO: assert Regions are sorted descending by pressure
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST); const auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
<< ", current = " << Occ << '\n'); << ", current = " << Occ << '\n');
auto NewOcc = TargetOcc; auto NewOcc = TargetOcc;
for (auto R : Regions) { for (auto R : Regions) {
if (R->MaxPressure.getOccupancy(ST) >= NewOcc) if (R->MaxPressure.getOccupancy(ST) >= NewOcc)
break; break;
Show All 20 Lines if (NewOcc > Occ) {
MFI->increaseOccupancy(MF, NewOcc); MFI->increaseOccupancy(MF, NewOcc);
} }
return std::max(NewOcc, Occ); return std::max(NewOcc, Occ);
} }
void GCNIterativeScheduler::scheduleLegacyMaxOccupancy( void GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
bool TryMaximizeOccupancy) { bool TryMaximizeOccupancy) {
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
auto TgtOcc = MFI->getMinAllowedOccupancy(); auto TgtOcc = MFI->getMinAllowedOccupancy();
sortRegionsByPressure(TgtOcc); sortRegionsByPressure(TgtOcc);
auto Occ = Regions.front()->MaxPressure.getOccupancy(ST); auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
if (TryMaximizeOccupancy && Occ < TgtOcc) if (TryMaximizeOccupancy && Occ < TgtOcc)
Occ = tryMaximizeOccupancy(TgtOcc); Occ = tryMaximizeOccupancy(TgtOcc);
Show All 37 Linesvoid GCNIterativeScheduler::scheduleLegacyMaxOccupancy(
} }
MFI->limitOccupancy(FinalOccupancy); MFI->limitOccupancy(FinalOccupancy);
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// Minimal Register Strategy // Minimal Register Strategy
void GCNIterativeScheduler::scheduleMinReg(bool force) { void GCNIterativeScheduler::scheduleMinReg(bool force) {
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
const auto TgtOcc = MFI->getOccupancy(); const auto TgtOcc = MFI->getOccupancy();
sortRegionsByPressure(TgtOcc); sortRegionsByPressure(TgtOcc);
auto MaxPressure = Regions.front()->MaxPressure; auto MaxPressure = Regions.front()->MaxPressure;
for (auto R : Regions) { for (auto R : Regions) {
if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc)) if (!force && R->MaxPressure.less(ST, MaxPressure, TgtOcc))
break; break;
Show All 17 Linesvoid GCNIterativeScheduler::scheduleMinReg(bool force) {
} }
} }
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// ILP scheduler port // ILP scheduler port
void GCNIterativeScheduler::scheduleILP( void GCNIterativeScheduler::scheduleILP(
bool TryMaximizeOccupancy) { bool TryMaximizeOccupancy) {
const auto &ST = MF.getSubtarget<SISubtarget>(); const auto &ST = MF.getSubtarget<GCNSubtarget>();
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
auto TgtOcc = MFI->getMinAllowedOccupancy(); auto TgtOcc = MFI->getMinAllowedOccupancy();
sortRegionsByPressure(TgtOcc); sortRegionsByPressure(TgtOcc);
auto Occ = Regions.front()->MaxPressure.getOccupancy(ST); auto Occ = Regions.front()->MaxPressure.getOccupancy(ST);
if (TryMaximizeOccupancy && Occ < TgtOcc) if (TryMaximizeOccupancy && Occ < TgtOcc)
Occ = tryMaximizeOccupancy(TgtOcc); Occ = tryMaximizeOccupancy(TgtOcc);
Show All 29 Lines

lib/Target/AMDGPU/GCNRegPressure.h

Show First 20 Lines • Show All 43 Lines • ▼ Show 20 Linesstruct GCNRegPressure {
void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); } void clear() { std::fill(&Value[0], &Value[TOTAL_KINDS], 0); }
unsigned getSGPRNum() const { return Value[SGPR32]; } unsigned getSGPRNum() const { return Value[SGPR32]; }
unsigned getVGPRNum() const { return Value[VGPR32]; } unsigned getVGPRNum() const { return Value[VGPR32]; }
unsigned getVGPRTuplesWeight() const { return Value[VGPR_TUPLE]; } unsigned getVGPRTuplesWeight() const { return Value[VGPR_TUPLE]; }
unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; } unsigned getSGPRTuplesWeight() const { return Value[SGPR_TUPLE]; }
unsigned getOccupancy(const SISubtarget &ST) const { unsigned getOccupancy(const GCNSubtarget &ST) const {
return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()), return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),
ST.getOccupancyWithNumVGPRs(getVGPRNum())); ST.getOccupancyWithNumVGPRs(getVGPRNum()));
} }
void inc(unsigned Reg, void inc(unsigned Reg,
LaneBitmask PrevMask, LaneBitmask PrevMask,
LaneBitmask NewMask, LaneBitmask NewMask,
const MachineRegisterInfo &MRI); const MachineRegisterInfo &MRI);
bool higherOccupancy(const SISubtarget &ST, const GCNRegPressure& O) const { bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure& O) const {
return getOccupancy(ST) > O.getOccupancy(ST); return getOccupancy(ST) > O.getOccupancy(ST);
} }
bool less(const SISubtarget &ST, const GCNRegPressure& O, bool less(const GCNSubtarget &ST, const GCNRegPressure& O,
unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const; unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;
bool operator==(const GCNRegPressure &O) const { bool operator==(const GCNRegPressure &O) const {
return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value); return std::equal(&Value[0], &Value[TOTAL_KINDS], O.Value);
} }
bool operator!=(const GCNRegPressure &O) const { bool operator!=(const GCNRegPressure &O) const {
return !(*this == O); return !(*this == O);
} }
void print(raw_ostream &OS, const SISubtarget *ST = nullptr) const; void print(raw_ostream &OS, const GCNSubtarget *ST = nullptr) const;
void dump() const { print(dbgs()); } void dump() const { print(dbgs()); }
private: private:
unsigned Value[TOTAL_KINDS]; unsigned Value[TOTAL_KINDS];
static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI); static unsigned getRegKind(unsigned Reg, const MachineRegisterInfo &MRI);
friend GCNRegPressure max(const GCNRegPressure &P1, friend GCNRegPressure max(const GCNRegPressure &P1,
▲ Show 20 Lines • Show All 136 Lines • Show Last 20 Lines

lib/Target/AMDGPU/GCNRegPressure.cpp

Show First 20 Lines • Show All 126 Lines • ▼ Show 20 Lines if (PrevMask.none()) {
Value[Kind] += Sign * MRI.getPressureSets(Reg).getWeight(); Value[Kind] += Sign * MRI.getPressureSets(Reg).getWeight();
} }
break; break;
default: llvm_unreachable("Unknown register kind"); default: llvm_unreachable("Unknown register kind");
} }
} }
bool GCNRegPressure::less(const SISubtarget &ST, bool GCNRegPressure::less(const GCNSubtarget &ST,
const GCNRegPressure& O, const GCNRegPressure& O,
unsigned MaxOccupancy) const { unsigned MaxOccupancy) const {
const auto SGPROcc = std::min(MaxOccupancy, const auto SGPROcc = std::min(MaxOccupancy,
ST.getOccupancyWithNumSGPRs(getSGPRNum())); ST.getOccupancyWithNumSGPRs(getSGPRNum()));
const auto VGPROcc = std::min(MaxOccupancy, const auto VGPROcc = std::min(MaxOccupancy,
ST.getOccupancyWithNumVGPRs(getVGPRNum())); ST.getOccupancyWithNumVGPRs(getVGPRNum()));
const auto OtherSGPROcc = std::min(MaxOccupancy, const auto OtherSGPROcc = std::min(MaxOccupancy,
ST.getOccupancyWithNumSGPRs(O.getSGPRNum())); ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));
Show All 29 Lines for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {
} }
} }
return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()): return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):
(getVGPRNum() < O.getVGPRNum()); (getVGPRNum() < O.getVGPRNum());
} }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD LLVM_DUMP_METHOD
void GCNRegPressure::print(raw_ostream &OS, const SISubtarget *ST) const { void GCNRegPressure::print(raw_ostream &OS, const GCNSubtarget *ST) const {
OS << "VGPRs: " << getVGPRNum(); OS << "VGPRs: " << getVGPRNum();
if (ST) OS << "(O" << ST->getOccupancyWithNumVGPRs(getVGPRNum()) << ')'; if (ST) OS << "(O" << ST->getOccupancyWithNumVGPRs(getVGPRNum()) << ')';
OS << ", SGPRs: " << getSGPRNum(); OS << ", SGPRs: " << getSGPRNum();
if (ST) OS << "(O" << ST->getOccupancyWithNumSGPRs(getSGPRNum()) << ')'; if (ST) OS << "(O" << ST->getOccupancyWithNumSGPRs(getSGPRNum()) << ')';
OS << ", LVGPR WT: " << getVGPRTuplesWeight() OS << ", LVGPR WT: " << getVGPRTuplesWeight()
<< ", LSGPR WT: " << getSGPRTuplesWeight(); << ", LSGPR WT: " << getSGPRTuplesWeight();
if (ST) OS << " -> Occ: " << getOccupancy(*ST); if (ST) OS << " -> Occ: " << getOccupancy(*ST);
OS << '\n'; OS << '\n';
▲ Show 20 Lines • Show All 318 Lines • Show Last 20 Lines

lib/Target/AMDGPU/GCNSchedStrategy.h

Show All 15 Lines
#include "GCNRegPressure.h" #include "GCNRegPressure.h"
#include "llvm/CodeGen/MachineScheduler.h" #include "llvm/CodeGen/MachineScheduler.h"
namespace llvm { namespace llvm {
class SIMachineFunctionInfo; class SIMachineFunctionInfo;
class SIRegisterInfo; class SIRegisterInfo;
class SISubtarget; class GCNSubtarget;
/// This is a minimal scheduler strategy. The main difference between this /// This is a minimal scheduler strategy. The main difference between this
/// and the GenericScheduler is that GCNSchedStrategy uses different /// and the GenericScheduler is that GCNSchedStrategy uses different
/// heuristics to determine excess/critical pressure sets. Its goal is to /// heuristics to determine excess/critical pressure sets. Its goal is to
/// maximize kernel occupancy (i.e. maximum number of waves per simd). /// maximize kernel occupancy (i.e. maximum number of waves per simd).
class GCNMaxOccupancySchedStrategy : public GenericScheduler { class GCNMaxOccupancySchedStrategy : public GenericScheduler {
friend class GCNScheduleDAGMILive; friend class GCNScheduleDAGMILive;
Show All 24 Linespublic:
void initialize(ScheduleDAGMI *DAG) override; void initialize(ScheduleDAGMI *DAG) override;
void setTargetOccupancy(unsigned Occ) { TargetOccupancy = Occ; } void setTargetOccupancy(unsigned Occ) { TargetOccupancy = Occ; }
}; };
class GCNScheduleDAGMILive : public ScheduleDAGMILive { class GCNScheduleDAGMILive : public ScheduleDAGMILive {
const SISubtarget &ST; const GCNSubtarget &ST;
SIMachineFunctionInfo &MFI; SIMachineFunctionInfo &MFI;
// Occupancy target at the beginning of function scheduling cycle. // Occupancy target at the beginning of function scheduling cycle.
unsigned StartingOccupancy; unsigned StartingOccupancy;
// Minimal real occupancy recorder for the function. // Minimal real occupancy recorder for the function.
unsigned MinOccupancy; unsigned MinOccupancy;
Show All 39 Lines

lib/Target/AMDGPU/GCNSchedStrategy.cpp

Show All 29 Lines
void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) { void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
GenericScheduler::initialize(DAG); GenericScheduler::initialize(DAG);
const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI); const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
MF = &DAG->MF; MF = &DAG->MF;
const SISubtarget &ST = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
// FIXME: This is also necessary, because some passes that run after // FIXME: This is also necessary, because some passes that run after
// scheduling and before regalloc increase register pressure. // scheduling and before regalloc increase register pressure.
const int ErrorMargin = 3; const int ErrorMargin = 3;
SGPRExcessLimit = Context->RegClassInfo SGPRExcessLimit = Context->RegClassInfo
->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin; ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
VGPRExcessLimit = Context->RegClassInfo VGPRExcessLimit = Context->RegClassInfo
▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 LinesSUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
<< *SU->getInstr()); << *SU->getInstr());
return SU; return SU;
} }
GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C, GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
std::unique_ptr<MachineSchedStrategy> S) : std::unique_ptr<MachineSchedStrategy> S) :
ScheduleDAGMILive(C, std::move(S)), ScheduleDAGMILive(C, std::move(S)),
ST(MF.getSubtarget<SISubtarget>()), ST(MF.getSubtarget<GCNSubtarget>()),
MFI(*MF.getInfo<SIMachineFunctionInfo>()), MFI(*MF.getInfo<SIMachineFunctionInfo>()),
StartingOccupancy(MFI.getOccupancy()), StartingOccupancy(MFI.getOccupancy()),
MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) { MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n"); LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
} }
void GCNScheduleDAGMILive::schedule() { void GCNScheduleDAGMILive::schedule() {
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines

lib/Target/AMDGPU/R600RegisterInfo.h

Show All 14 Lines
#ifndef LLVM_LIB_TARGET_AMDGPU_R600REGISTERINFO_H #ifndef LLVM_LIB_TARGET_AMDGPU_R600REGISTERINFO_H
#define LLVM_LIB_TARGET_AMDGPU_R600REGISTERINFO_H #define LLVM_LIB_TARGET_AMDGPU_R600REGISTERINFO_H
#define GET_REGINFO_HEADER #define GET_REGINFO_HEADER
#include "R600GenRegisterInfo.inc" #include "R600GenRegisterInfo.inc"
namespace llvm { namespace llvm {
class AMDGPUSubtarget; class GCNSubtarget;
jveselyUnsubmitted
Done
ReplyInline Actions

Why is this necessary in R600 file? Looks like a leftover from the time when AMDGPUSubtarget was shared parent.

jvesely: Why is this necessary in R600 file? Looks like a leftover from the time when AMDGPUSubtarget…
struct R600RegisterInfo final : public R600GenRegisterInfo { struct R600RegisterInfo final : public R600GenRegisterInfo {
RegClassWeight RCW; RegClassWeight RCW;
R600RegisterInfo(); R600RegisterInfo();
BitVector getReservedRegs(const MachineFunction &MF) const override; BitVector getReservedRegs(const MachineFunction &MF) const override;
const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override; const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
Show All 28 Lines

lib/Target/AMDGPU/SIDebuggerInsertNops.cpp

Show First 20 Lines • Show All 57 Lines • ▼ Show 20 Lines
FunctionPass *llvm::createSIDebuggerInsertNopsPass() { FunctionPass *llvm::createSIDebuggerInsertNopsPass() {
return new SIDebuggerInsertNops(); return new SIDebuggerInsertNops();
} }
bool SIDebuggerInsertNops::runOnMachineFunction(MachineFunction &MF) { bool SIDebuggerInsertNops::runOnMachineFunction(MachineFunction &MF) {
// Skip this pass if "amdgpu-debugger-insert-nops" attribute was not // Skip this pass if "amdgpu-debugger-insert-nops" attribute was not
// specified. // specified.
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (!ST.debuggerInsertNops()) if (!ST.debuggerInsertNops())
return false; return false;
// Skip machine functions without debug info. // Skip machine functions without debug info.
if (!MF.getMMI().hasDebugInfo()) if (!MF.getMMI().hasDebugInfo())
return false; return false;
// Target instruction info. // Target instruction info.
Show All 23 Lines

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

Show First 20 Lines • Show All 562 Lines • ▼ Show 20 Linesstatic bool hoistAndMergeSGPRInits(unsigned Reg,
if (Changed) if (Changed)
MRI.clearKillFlags(Reg); MRI.clearKillFlags(Reg);
return Changed; return Changed;
} }
bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) { bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
MDT = &getAnalysis<MachineDominatorTree>(); MDT = &getAnalysis<MachineDominatorTree>();
SmallVector<MachineInstr *, 16> Worklist; SmallVector<MachineInstr *, 16> Worklist;
for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
▲ Show 20 Lines • Show All 136 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIFixVGPRCopies.cpp

Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Lines
INITIALIZE_PASS(SIFixVGPRCopies, DEBUG_TYPE, "SI Fix VGPR copies", false, false) INITIALIZE_PASS(SIFixVGPRCopies, DEBUG_TYPE, "SI Fix VGPR copies", false, false)
char SIFixVGPRCopies::ID = 0; char SIFixVGPRCopies::ID = 0;
char &llvm::SIFixVGPRCopiesID = SIFixVGPRCopies::ID; char &llvm::SIFixVGPRCopiesID = SIFixVGPRCopies::ID;
bool SIFixVGPRCopies::runOnMachineFunction(MachineFunction &MF) { bool SIFixVGPRCopies::runOnMachineFunction(MachineFunction &MF) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
bool Changed = false; bool Changed = false;
for (MachineBasicBlock &MBB : MF) { for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) { for (MachineInstr &MI : MBB) {
switch (MI.getOpcode()) { switch (MI.getOpcode()) {
case AMDGPU::COPY: case AMDGPU::COPY:
Show All 15 Lines

lib/Target/AMDGPU/SIFixWWMLiveness.cpp

Show First 20 Lines • Show All 179 Lines • ▼ Show 20 Lines
} }
bool SIFixWWMLiveness::runOnMachineFunction(MachineFunction &MF) { bool SIFixWWMLiveness::runOnMachineFunction(MachineFunction &MF) {
bool Modified = false; bool Modified = false;
// This doesn't actually need LiveIntervals, but we can preserve them. // This doesn't actually need LiveIntervals, but we can preserve them.
LIS = getAnalysisIfAvailable<LiveIntervals>(); LIS = getAnalysisIfAvailable<LiveIntervals>();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
for (MachineBasicBlock &MBB : MF) { for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) { for (MachineInstr &MI : MBB) {
if (MI.getOpcode() == AMDGPU::EXIT_WWM) { if (MI.getOpcode() == AMDGPU::EXIT_WWM) {
Modified |= runOnWWMInstruction(MI); Modified |= runOnWWMInstruction(MI);
} }
} }
} }
return Modified; return Modified;
} }

lib/Target/AMDGPU/SIFoldOperands.cpp

Show First 20 Lines • Show All 70 Lines • ▼ Show 20 Lines
}; };
class SIFoldOperands : public MachineFunctionPass { class SIFoldOperands : public MachineFunctionPass {
public: public:
static char ID; static char ID;
MachineRegisterInfo *MRI; MachineRegisterInfo *MRI;
const SIInstrInfo *TII; const SIInstrInfo *TII;
const SIRegisterInfo *TRI; const SIRegisterInfo *TRI;
const AMDGPUSubtarget *ST; const GCNSubtarget *ST;
void foldOperand(MachineOperand &OpToFold, void foldOperand(MachineOperand &OpToFold,
MachineInstr *UseMI, MachineInstr *UseMI,
unsigned UseOpIdx, unsigned UseOpIdx,
SmallVectorImpl<FoldCandidate> &FoldList, SmallVectorImpl<FoldCandidate> &FoldList,
SmallVectorImpl<MachineInstr *> &CopiesToReplace) const; SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
void foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const; void foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;
▲ Show 20 Lines • Show All 879 Lines • ▼ Show 20 Linesbool SIFoldOperands::tryFoldOMod(MachineInstr &MI) {
return true; return true;
} }
bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) { bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction())) if (skipFunction(MF.getFunction()))
return false; return false;
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
ST = &MF.getSubtarget<AMDGPUSubtarget>(); ST = &MF.getSubtarget<GCNSubtarget>();
TII = ST->getInstrInfo(); TII = ST->getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
// omod is ignored by hardware if IEEE bit is enabled. omod also does not // omod is ignored by hardware if IEEE bit is enabled. omod also does not
// correctly handle signed zeros. // correctly handle signed zeros.
// //
▲ Show 20 Lines • Show All 45 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIFormMemoryClauses.cpp

Show First 20 Lines • Show All 64 Lines • ▼ Show 20 Linesprivate:
void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const; void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const; bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT); bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const; void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses, bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
GCNDownwardRPTracker &RPT); GCNDownwardRPTracker &RPT);
const SISubtarget *ST; const GCNSubtarget *ST;
const SIRegisterInfo *TRI; const SIRegisterInfo *TRI;
const MachineRegisterInfo *MRI; const MachineRegisterInfo *MRI;
SIMachineFunctionInfo *MFI; SIMachineFunctionInfo *MFI;
unsigned LastRecordedOccupancy; unsigned LastRecordedOccupancy;
unsigned MaxVGPRs; unsigned MaxVGPRs;
unsigned MaxSGPRs; unsigned MaxSGPRs;
}; };
▲ Show 20 Lines • Show All 209 Lines • ▼ Show 20 Linesbool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
collectRegUses(MI, Defs, Uses); collectRegUses(MI, Defs, Uses);
return true; return true;
} }
bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) { bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction())) if (skipFunction(MF.getFunction()))
return false; return false;
ST = &MF.getSubtarget<SISubtarget>(); ST = &MF.getSubtarget<GCNSubtarget>();
if (!ST->isXNACKEnabled()) if (!ST->isXNACKEnabled())
return false; return false;
const SIInstrInfo *TII = ST->getInstrInfo(); const SIInstrInfo *TII = ST->getInstrInfo();
TRI = ST->getRegisterInfo(); TRI = ST->getRegisterInfo();
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
MFI = MF.getInfo<SIMachineFunctionInfo>(); MFI = MF.getInfo<SIMachineFunctionInfo>();
LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
▲ Show 20 Lines • Show All 91 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIFrameLowering.h

Show All 11 Lines
#include "AMDGPUFrameLowering.h" #include "AMDGPUFrameLowering.h"
namespace llvm { namespace llvm {
class SIInstrInfo; class SIInstrInfo;
class SIMachineFunctionInfo; class SIMachineFunctionInfo;
class SIRegisterInfo; class SIRegisterInfo;
class SISubtarget; class GCNSubtarget;
class SIFrameLowering final : public AMDGPUFrameLowering { class SIFrameLowering final : public AMDGPUFrameLowering {
public: public:
SIFrameLowering(StackDirection D, unsigned StackAl, int LAO, SIFrameLowering(StackDirection D, unsigned StackAl, int LAO,
unsigned TransAl = 1) : unsigned TransAl = 1) :
AMDGPUFrameLowering(D, StackAl, LAO, TransAl) {} AMDGPUFrameLowering(D, StackAl, LAO, TransAl) {}
~SIFrameLowering() override = default; ~SIFrameLowering() override = default;
Show All 14 Lines void processFunctionBeforeFrameFinalized(
RegScavenger *RS = nullptr) const override; RegScavenger *RS = nullptr) const override;
MachineBasicBlock::iterator MachineBasicBlock::iterator
eliminateCallFramePseudoInstr(MachineFunction &MF, eliminateCallFramePseudoInstr(MachineFunction &MF,
MachineBasicBlock &MBB, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI) const override; MachineBasicBlock::iterator MI) const override;
private: private:
void emitFlatScratchInit(const SISubtarget &ST, void emitFlatScratchInit(const GCNSubtarget &ST,
MachineFunction &MF, MachineFunction &MF,
MachineBasicBlock &MBB) const; MachineBasicBlock &MBB) const;
unsigned getReservedPrivateSegmentBufferReg( unsigned getReservedPrivateSegmentBufferReg(
const SISubtarget &ST, const GCNSubtarget &ST,
const SIInstrInfo *TII, const SIInstrInfo *TII,
const SIRegisterInfo *TRI, const SIRegisterInfo *TRI,
SIMachineFunctionInfo *MFI, SIMachineFunctionInfo *MFI,
MachineFunction &MF) const; MachineFunction &MF) const;
std::pair<unsigned, unsigned> getReservedPrivateSegmentWaveByteOffsetReg( std::pair<unsigned, unsigned> getReservedPrivateSegmentWaveByteOffsetReg(
const SISubtarget &ST, const GCNSubtarget &ST,
const SIInstrInfo *TII, const SIInstrInfo *TII,
const SIRegisterInfo *TRI, const SIRegisterInfo *TRI,
SIMachineFunctionInfo *MFI, SIMachineFunctionInfo *MFI,
MachineFunction &MF) const; MachineFunction &MF) const;
/// Emits debugger prologue. /// Emits debugger prologue.
void emitDebuggerPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const; void emitDebuggerPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const;
// Emit scratch setup code for AMDPAL or Mesa, assuming ResourceRegUsed is set. // Emit scratch setup code for AMDPAL or Mesa, assuming ResourceRegUsed is set.
void emitEntryFunctionScratchSetup(const SISubtarget &ST, MachineFunction &MF, void emitEntryFunctionScratchSetup(const GCNSubtarget &ST, MachineFunction &MF,
MachineBasicBlock &MBB, SIMachineFunctionInfo *MFI, MachineBasicBlock &MBB, SIMachineFunctionInfo *MFI,
MachineBasicBlock::iterator I, unsigned PreloadedPrivateBufferReg, MachineBasicBlock::iterator I, unsigned PreloadedPrivateBufferReg,
unsigned ScratchRsrcReg) const; unsigned ScratchRsrcReg) const;
public: public:
bool hasFP(const MachineFunction &MF) const override; bool hasFP(const MachineFunction &MF) const override;
bool hasSP(const MachineFunction &MF) const; bool hasSP(const MachineFunction &MF) const;
}; };
} // end namespace llvm } // end namespace llvm
#endif // LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H #endif // LLVM_LIB_TARGET_AMDGPU_SIFRAMELOWERING_H

lib/Target/AMDGPU/SIFrameLowering.cpp

Show All 17 Lines
#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/RegisterScavenging.h"
using namespace llvm; using namespace llvm;
static ArrayRef<MCPhysReg> getAllSGPR128(const SISubtarget &ST, static ArrayRef<MCPhysReg> getAllSGPR128(const GCNSubtarget &ST,
const MachineFunction &MF) { const MachineFunction &MF) {
return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(), return makeArrayRef(AMDGPU::SGPR_128RegClass.begin(),
ST.getMaxNumSGPRs(MF) / 4); ST.getMaxNumSGPRs(MF) / 4);
} }
static ArrayRef<MCPhysReg> getAllSGPRs(const SISubtarget &ST, static ArrayRef<MCPhysReg> getAllSGPRs(const GCNSubtarget &ST,
const MachineFunction &MF) { const MachineFunction &MF) {
return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(), return makeArrayRef(AMDGPU::SGPR_32RegClass.begin(),
ST.getMaxNumSGPRs(MF)); ST.getMaxNumSGPRs(MF));
} }
void SIFrameLowering::emitFlatScratchInit(const SISubtarget &ST, void SIFrameLowering::emitFlatScratchInit(const GCNSubtarget &ST,
MachineFunction &MF, MachineFunction &MF,
MachineBasicBlock &MBB) const { MachineBasicBlock &MBB) const {
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo* TRI = &TII->getRegisterInfo(); const SIRegisterInfo* TRI = &TII->getRegisterInfo();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
// We don't need this if we only have spills since there is no user facing // We don't need this if we only have spills since there is no user facing
// scratch. // scratch.
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Linesvoid SIFrameLowering::emitFlatScratchInit(const GCNSubtarget &ST,
// Convert offset to 256-byte units. // Convert offset to 256-byte units.
BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI) BuildMI(MBB, I, DL, TII->get(AMDGPU::S_LSHR_B32), AMDGPU::FLAT_SCR_HI)
.addReg(FlatScrInitLo, RegState::Kill) .addReg(FlatScrInitLo, RegState::Kill)
.addImm(8); .addImm(8);
} }
unsigned SIFrameLowering::getReservedPrivateSegmentBufferReg( unsigned SIFrameLowering::getReservedPrivateSegmentBufferReg(
const SISubtarget &ST, const GCNSubtarget &ST,
const SIInstrInfo *TII, const SIInstrInfo *TII,
const SIRegisterInfo *TRI, const SIRegisterInfo *TRI,
SIMachineFunctionInfo *MFI, SIMachineFunctionInfo *MFI,
MachineFunction &MF) const { MachineFunction &MF) const {
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
// We need to insert initialization of the scratch resource descriptor. // We need to insert initialization of the scratch resource descriptor.
unsigned ScratchRsrcReg = MFI->getScratchRSrcReg(); unsigned ScratchRsrcReg = MFI->getScratchRSrcReg();
Show All 34 Linesunsigned SIFrameLowering::getReservedPrivateSegmentBufferReg(
return ScratchRsrcReg; return ScratchRsrcReg;
} }
// Shift down registers reserved for the scratch wave offset and stack pointer // Shift down registers reserved for the scratch wave offset and stack pointer
// SGPRs. // SGPRs.
std::pair<unsigned, unsigned> std::pair<unsigned, unsigned>
SIFrameLowering::getReservedPrivateSegmentWaveByteOffsetReg( SIFrameLowering::getReservedPrivateSegmentWaveByteOffsetReg(
const SISubtarget &ST, const GCNSubtarget &ST,
const SIInstrInfo *TII, const SIInstrInfo *TII,
const SIRegisterInfo *TRI, const SIRegisterInfo *TRI,
SIMachineFunctionInfo *MFI, SIMachineFunctionInfo *MFI,
MachineFunction &MF) const { MachineFunction &MF) const {
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg(); unsigned ScratchWaveOffsetReg = MFI->getScratchWaveOffsetReg();
// No replacement necessary. // No replacement necessary.
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 LinesSIFrameLowering::getReservedPrivateSegmentWaveByteOffsetReg(
return std::make_pair(ScratchWaveOffsetReg, SPReg); return std::make_pair(ScratchWaveOffsetReg, SPReg);
} }
void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF, void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
MachineBasicBlock &MBB) const { MachineBasicBlock &MBB) const {
// Emit debugger prologue if "amdgpu-debugger-emit-prologue" attribute was // Emit debugger prologue if "amdgpu-debugger-emit-prologue" attribute was
// specified. // specified.
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (ST.debuggerEmitPrologue()) if (ST.debuggerEmitPrologue())
emitDebuggerPrologue(MF, MBB); emitDebuggerPrologue(MF, MBB);
assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
// If we only have SGPR spills, we won't actually be using scratch memory // If we only have SGPR spills, we won't actually be using scratch memory
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Linesvoid SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
} }
if (ResourceRegUsed) if (ResourceRegUsed)
emitEntryFunctionScratchSetup(ST, MF, MBB, MFI, I, emitEntryFunctionScratchSetup(ST, MF, MBB, MFI, I,
PreloadedPrivateBufferReg, ScratchRsrcReg); PreloadedPrivateBufferReg, ScratchRsrcReg);
} }
// Emit scratch setup code for AMDPAL or Mesa, assuming ResourceRegUsed is set. // Emit scratch setup code for AMDPAL or Mesa, assuming ResourceRegUsed is set.
void SIFrameLowering::emitEntryFunctionScratchSetup(const SISubtarget &ST, void SIFrameLowering::emitEntryFunctionScratchSetup(const GCNSubtarget &ST,
MachineFunction &MF, MachineBasicBlock &MBB, SIMachineFunctionInfo *MFI, MachineFunction &MF, MachineBasicBlock &MBB, SIMachineFunctionInfo *MFI,
MachineBasicBlock::iterator I, unsigned PreloadedPrivateBufferReg, MachineBasicBlock::iterator I, unsigned PreloadedPrivateBufferReg,
unsigned ScratchRsrcReg) const { unsigned ScratchRsrcReg) const {
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo *TRI = &TII->getRegisterInfo(); const SIRegisterInfo *TRI = &TII->getRegisterInfo();
const Function &Fn = MF.getFunction(); const Function &Fn = MF.getFunction();
DebugLoc DL; DebugLoc DL;
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines
// FIXME: This is a bit conservative, since in the above case we could use one // FIXME: This is a bit conservative, since in the above case we could use one
// of the callee-save registers as a scratch temp to re-align the stack pointer, // of the callee-save registers as a scratch temp to re-align the stack pointer,
// but we would then have to make sure that we were in fact saving at least one // but we would then have to make sure that we were in fact saving at least one
// callee-save register in the prologue, which is additional complexity that // callee-save register in the prologue, which is additional complexity that
// doesn't seem worth the benefit. // doesn't seem worth the benefit.
static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock &MBB) { static unsigned findScratchNonCalleeSaveRegister(MachineBasicBlock &MBB) {
MachineFunction *MF = MBB.getParent(); MachineFunction *MF = MBB.getParent();
const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>();
const SIRegisterInfo &TRI = *Subtarget.getRegisterInfo(); const SIRegisterInfo &TRI = *Subtarget.getRegisterInfo();
LivePhysRegs LiveRegs(TRI); LivePhysRegs LiveRegs(TRI);
LiveRegs.addLiveIns(MBB); LiveRegs.addLiveIns(MBB);
// Mark callee saved registers as used so we will not choose them. // Mark callee saved registers as used so we will not choose them.
const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF); const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF);
for (unsigned i = 0; CSRegs[i]; ++i) for (unsigned i = 0; CSRegs[i]; ++i)
LiveRegs.addReg(CSRegs[i]); LiveRegs.addReg(CSRegs[i]);
Show All 12 Linesvoid SIFrameLowering::emitPrologue(MachineFunction &MF,
MachineBasicBlock &MBB) const { MachineBasicBlock &MBB) const {
SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
if (FuncInfo->isEntryFunction()) { if (FuncInfo->isEntryFunction()) {
emitEntryFunctionPrologue(MF, MBB); emitEntryFunctionPrologue(MF, MBB);
return; return;
} }
const MachineFrameInfo &MFI = MF.getFrameInfo(); const MachineFrameInfo &MFI = MF.getFrameInfo();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo &TRI = TII->getRegisterInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo();
unsigned StackPtrReg = FuncInfo->getStackPtrOffsetReg(); unsigned StackPtrReg = FuncInfo->getStackPtrOffsetReg();
unsigned FramePtrReg = FuncInfo->getFrameOffsetReg(); unsigned FramePtrReg = FuncInfo->getFrameOffsetReg();
MachineBasicBlock::iterator MBBI = MBB.begin(); MachineBasicBlock::iterator MBBI = MBB.begin();
DebugLoc DL; DebugLoc DL;
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines
} }
void SIFrameLowering::emitEpilogue(MachineFunction &MF, void SIFrameLowering::emitEpilogue(MachineFunction &MF,
MachineBasicBlock &MBB) const { MachineBasicBlock &MBB) const {
const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
if (FuncInfo->isEntryFunction()) if (FuncInfo->isEntryFunction())
return; return;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(); MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg
: FuncInfo->getSGPRSpillVGPRs()) { : FuncInfo->getSGPRSpillVGPRs()) {
if (!Reg.FI.hasValue()) if (!Reg.FI.hasValue())
continue; continue;
TII->loadRegFromStackSlot(MBB, MBBI, Reg.VGPR, TII->loadRegFromStackSlot(MBB, MBBI, Reg.VGPR,
Show All 30 Lines if (!MFI.isDeadObjectIndex(I))
return false; return false;
} }
return true; return true;
} }
int SIFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, int SIFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
unsigned &FrameReg) const { unsigned &FrameReg) const {
const SIRegisterInfo *RI = MF.getSubtarget<SISubtarget>().getRegisterInfo(); const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
FrameReg = RI->getFrameRegister(MF); FrameReg = RI->getFrameRegister(MF);
return MF.getFrameInfo().getObjectOffset(FI); return MF.getFrameInfo().getObjectOffset(FI);
} }
void SIFrameLowering::processFunctionBeforeFrameFinalized( void SIFrameLowering::processFunctionBeforeFrameFinalized(
MachineFunction &MF, MachineFunction &MF,
RegScavenger *RS) const { RegScavenger *RS) const {
MachineFrameInfo &MFI = MF.getFrameInfo(); MachineFrameInfo &MFI = MF.getFrameInfo();
if (!MFI.hasStackObjects()) if (!MFI.hasStackObjects())
return; return;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo &TRI = TII->getRegisterInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo();
SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
bool AllSGPRSpilledToVGPRs = false; bool AllSGPRSpilledToVGPRs = false;
if (TRI.spillSGPRToVGPR() && FuncInfo->hasSpilledSGPRs()) { if (TRI.spillSGPRToVGPR() && FuncInfo->hasSpilledSGPRs()) {
AllSGPRSpilledToVGPRs = true; AllSGPRSpilledToVGPRs = true;
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines
MachineBasicBlock::iterator SIFrameLowering::eliminateCallFramePseudoInstr( MachineBasicBlock::iterator SIFrameLowering::eliminateCallFramePseudoInstr(
MachineFunction &MF, MachineFunction &MF,
MachineBasicBlock &MBB, MachineBasicBlock &MBB,
MachineBasicBlock::iterator I) const { MachineBasicBlock::iterator I) const {
int64_t Amount = I->getOperand(0).getImm(); int64_t Amount = I->getOperand(0).getImm();
if (Amount == 0) if (Amount == 0)
return MBB.erase(I); return MBB.erase(I);
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const DebugLoc &DL = I->getDebugLoc(); const DebugLoc &DL = I->getDebugLoc();
unsigned Opc = I->getOpcode(); unsigned Opc = I->getOpcode();
bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode(); bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0; uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
if (!TFI->hasReservedCallFrame(MF)) { if (!TFI->hasReservedCallFrame(MF)) {
Show All 12 Lines if (!TFI->hasReservedCallFrame(MF)) {
llvm_unreachable("is this used?"); llvm_unreachable("is this used?");
} }
return MBB.erase(I); return MBB.erase(I);
} }
void SIFrameLowering::emitDebuggerPrologue(MachineFunction &MF, void SIFrameLowering::emitDebuggerPrologue(MachineFunction &MF,
MachineBasicBlock &MBB) const { MachineBasicBlock &MBB) const {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo *TRI = &TII->getRegisterInfo(); const SIRegisterInfo *TRI = &TII->getRegisterInfo();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
MachineBasicBlock::iterator I = MBB.begin(); MachineBasicBlock::iterator I = MBB.begin();
DebugLoc DL; DebugLoc DL;
// For each dimension: // For each dimension:
Show All 33 Linesbool SIFrameLowering::hasFP(const MachineFunction &MF) const {
const MachineFrameInfo &MFI = MF.getFrameInfo(); const MachineFrameInfo &MFI = MF.getFrameInfo();
// XXX - Is this only called after frame is finalized? Should be able to check // XXX - Is this only called after frame is finalized? Should be able to check
// frame size. // frame size.
return MFI.hasStackObjects() && !allStackObjectsAreDead(MFI); return MFI.hasStackObjects() && !allStackObjectsAreDead(MFI);
} }
bool SIFrameLowering::hasSP(const MachineFunction &MF) const { bool SIFrameLowering::hasSP(const MachineFunction &MF) const {
const SIRegisterInfo *TRI = MF.getSubtarget<SISubtarget>().getRegisterInfo(); const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
// All stack operations are relative to the frame offset SGPR. // All stack operations are relative to the frame offset SGPR.
const MachineFrameInfo &MFI = MF.getFrameInfo(); const MachineFrameInfo &MFI = MF.getFrameInfo();
return MFI.hasCalls() || MFI.hasVarSizedObjects() || TRI->needsStackRealignment(MF); return MFI.hasCalls() || MFI.hasVarSizedObjects() || TRI->needsStackRealignment(MF);
} }

lib/Target/AMDGPU/SIISelLowering.h

Show All 17 Lines
#include "AMDGPUISelLowering.h" #include "AMDGPUISelLowering.h"
#include "AMDGPUArgumentUsageInfo.h" #include "AMDGPUArgumentUsageInfo.h"
#include "SIInstrInfo.h" #include "SIInstrInfo.h"
namespace llvm { namespace llvm {
class SITargetLowering final : public AMDGPUTargetLowering { class SITargetLowering final : public AMDGPUTargetLowering {
private: private:
const SISubtarget *Subtarget; const GCNSubtarget *Subtarget;
SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL, SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL,
SDValue Chain, uint64_t Offset) const; SDValue Chain, uint64_t Offset) const;
SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const; SDValue getImplicitArgPtr(SelectionDAG &DAG, const SDLoc &SL) const;
SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT, SDValue lowerKernargMemParameter(SelectionDAG &DAG, EVT VT, EVT MemVT,
const SDLoc &SL, SDValue Chain, const SDLoc &SL, SDValue Chain,
uint64_t Offset, unsigned Align, bool Signed, uint64_t Offset, unsigned Align, bool Signed,
const ISD::InputArg *Arg = nullptr) const; const ISD::InputArg *Arg = nullptr) const;
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Linesprivate:
/// \p GV, false otherwise. /// \p GV, false otherwise.
bool shouldEmitGOTReloc(const GlobalValue *GV) const; bool shouldEmitGOTReloc(const GlobalValue *GV) const;
/// \returns True if PC-relative relocation needs to be emitted for given /// \returns True if PC-relative relocation needs to be emitted for given
/// global value \p GV, false otherwise. /// global value \p GV, false otherwise.
bool shouldEmitPCReloc(const GlobalValue *GV) const; bool shouldEmitPCReloc(const GlobalValue *GV) const;
public: public:
SITargetLowering(const TargetMachine &tm, const SISubtarget &STI); SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI);
const SISubtarget *getSubtarget() const; const GCNSubtarget *getSubtarget() const;
bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const override; bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const override;
bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override; bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override;
bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
MachineFunction &MF, MachineFunction &MF,
unsigned IntrinsicID) const override; unsigned IntrinsicID) const override;
▲ Show 20 Lines • Show All 141 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 106 Lines • ▼ Show 20 Lines for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) { if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
return AMDGPU::SGPR0 + Reg; return AMDGPU::SGPR0 + Reg;
} }
} }
llvm_unreachable("Cannot allocate sgpr"); llvm_unreachable("Cannot allocate sgpr");
} }
SITargetLowering::SITargetLowering(const TargetMachine &TM, SITargetLowering::SITargetLowering(const TargetMachine &TM,
const SISubtarget &STI) const GCNSubtarget &STI)
: AMDGPUTargetLowering(TM, STI), : AMDGPUTargetLowering(TM, STI),
Subtarget(&STI) { Subtarget(&STI) {
addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
addRegisterClass(MVT::i32, &AMDGPU::SReg_32_XM0RegClass); addRegisterClass(MVT::i32, &AMDGPU::SReg_32_XM0RegClass);
addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass);
▲ Show 20 Lines • Show All 249 Lines • ▼ Show 20 Lines#endif
// span the midpoint are probably relatively rare, so don't worry about them // span the midpoint are probably relatively rare, so don't worry about them
// for now. // for now.
if (Subtarget->hasBFE()) if (Subtarget->hasBFE())
setHasExtractBitsInsn(true); setHasExtractBitsInsn(true);
setOperationAction(ISD::FMINNUM, MVT::f64, Legal); setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
if (Subtarget->getGeneration() >= SISubtarget::SEA_ISLANDS) { if (Subtarget->getGeneration() >= GCNSubtarget::SEA_ISLANDS) {
setOperationAction(ISD::FTRUNC, MVT::f64, Legal); setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
setOperationAction(ISD::FCEIL, MVT::f64, Legal); setOperationAction(ISD::FCEIL, MVT::f64, Legal);
setOperationAction(ISD::FRINT, MVT::f64, Legal); setOperationAction(ISD::FRINT, MVT::f64, Legal);
} else { } else {
setOperationAction(ISD::FCEIL, MVT::f64, Custom); setOperationAction(ISD::FCEIL, MVT::f64, Custom);
setOperationAction(ISD::FTRUNC, MVT::f64, Custom); setOperationAction(ISD::FTRUNC, MVT::f64, Custom);
setOperationAction(ISD::FRINT, MVT::f64, Custom); setOperationAction(ISD::FRINT, MVT::f64, Custom);
setOperationAction(ISD::FFLOOR, MVT::f64, Custom); setOperationAction(ISD::FFLOOR, MVT::f64, Custom);
▲ Show 20 Lines • Show All 272 Lines • ▼ Show 20 Lines#endif
setSchedulingPreference(Sched::RegPressure); setSchedulingPreference(Sched::RegPressure);
// SI at least has hardware support for floating point exceptions, but no way // SI at least has hardware support for floating point exceptions, but no way
// of using or handling them is implemented. They are also optional in OpenCL // of using or handling them is implemented. They are also optional in OpenCL
// (Section 7.3) // (Section 7.3)
setHasFloatingPointExceptions(Subtarget->hasFPExceptions()); setHasFloatingPointExceptions(Subtarget->hasFPExceptions());
} }
const SISubtarget *SITargetLowering::getSubtarget() const { const GCNSubtarget *SITargetLowering::getSubtarget() const {
return Subtarget; return Subtarget;
} }
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// TargetLowering queries // TargetLowering queries
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// v_mad_mix* support a conversion from f16 to f32. // v_mad_mix* support a conversion from f16 to f32.
Show All 24 Lines AttributeList Attr = Intrinsic::getAttributes(CI.getContext(),
(Intrinsic::ID)IntrID); (Intrinsic::ID)IntrID);
if (Attr.hasFnAttribute(Attribute::ReadNone)) if (Attr.hasFnAttribute(Attribute::ReadNone))
return false; return false;
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
if (RsrcIntr->IsImage) { if (RsrcIntr->IsImage) {
Info.ptrVal = MFI->getImagePSV( Info.ptrVal = MFI->getImagePSV(
*MF.getSubtarget<SISubtarget>().getInstrInfo(), *MF.getSubtarget<GCNSubtarget>().getInstrInfo(),
CI.getArgOperand(RsrcIntr->RsrcArg)); CI.getArgOperand(RsrcIntr->RsrcArg));
Info.align = 0; Info.align = 0;
} else { } else {
Info.ptrVal = MFI->getBufferPSV( Info.ptrVal = MFI->getBufferPSV(
*MF.getSubtarget<SISubtarget>().getInstrInfo(), *MF.getSubtarget<GCNSubtarget>().getInstrInfo(),
CI.getArgOperand(RsrcIntr->RsrcArg)); CI.getArgOperand(RsrcIntr->RsrcArg));
} }
Info.flags = MachineMemOperand::MODereferenceable; Info.flags = MachineMemOperand::MODereferenceable;
if (Attr.hasFnAttribute(Attribute::ReadOnly)) { if (Attr.hasFnAttribute(Attribute::ReadOnly)) {
Info.opc = ISD::INTRINSIC_W_CHAIN; Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.memVT = MVT::getVT(CI.getType()); Info.memVT = MVT::getVT(CI.getType());
Info.flags |= MachineMemOperand::MOLoad; Info.flags |= MachineMemOperand::MOLoad;
▲ Show 20 Lines • Show All 147 Lines • ▼ Show 20 Lines if (AS == AMDGPUASI.CONSTANT_ADDRESS ||
// There are no SMRD extloads, so if we have to do a small type access we // There are no SMRD extloads, so if we have to do a small type access we
// will use a MUBUF load. // will use a MUBUF load.
// FIXME?: We also need to do this if unaligned, but we don't know the // FIXME?: We also need to do this if unaligned, but we don't know the
// alignment here. // alignment here.
if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4) if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
return isLegalGlobalAddressingMode(AM); return isLegalGlobalAddressingMode(AM);
if (Subtarget->getGeneration() == SISubtarget::SOUTHERN_ISLANDS) { if (Subtarget->getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS) {
// SMRD instructions have an 8-bit, dword offset on SI. // SMRD instructions have an 8-bit, dword offset on SI.
if (!isUInt<8>(AM.BaseOffs / 4)) if (!isUInt<8>(AM.BaseOffs / 4))
return false; return false;
} else if (Subtarget->getGeneration() == SISubtarget::SEA_ISLANDS) { } else if (Subtarget->getGeneration() == GCNSubtarget::SEA_ISLANDS) {
// On CI+, this can also be a 32-bit literal constant offset. If it fits // On CI+, this can also be a 32-bit literal constant offset. If it fits
// in 8-bits, it can use a smaller encoding. // in 8-bits, it can use a smaller encoding.
if (!isUInt<32>(AM.BaseOffs / 4)) if (!isUInt<32>(AM.BaseOffs / 4))
return false; return false;
} else if (Subtarget->getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { } else if (Subtarget->getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
// On VI, these use the SMEM format and the offset is 20-bit in bytes. // On VI, these use the SMEM format and the offset is 20-bit in bytes.
if (!isUInt<20>(AM.BaseOffs)) if (!isUInt<20>(AM.BaseOffs))
return false; return false;
} else } else
llvm_unreachable("unhandled generation"); llvm_unreachable("unhandled generation");
if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg. if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
return true; return true;
▲ Show 20 Lines • Show All 657 Lines • ▼ Show 20 Linesstatic void reservePrivateMemoryRegs(const TargetMachine &TM,
// almost certain that spilling will be required. // almost certain that spilling will be required.
if (TM.getOptLevel() == CodeGenOpt::None) if (TM.getOptLevel() == CodeGenOpt::None)
HasStackObjects = true; HasStackObjects = true;
// For now assume stack access is needed in any callee functions, so we need // For now assume stack access is needed in any callee functions, so we need
// the scratch registers to pass in. // the scratch registers to pass in.
bool RequiresStackAccess = HasStackObjects || MFI.hasCalls(); bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (ST.isAmdCodeObjectV2(MF.getFunction())) { if (ST.isAmdCodeObjectV2(MF.getFunction())) {
if (RequiresStackAccess) { if (RequiresStackAccess) {
// If we have stack objects, we unquestionably need the private buffer // If we have stack objects, we unquestionably need the private buffer
// resource. For the Code Object V2 ABI, this will be the first 4 user // resource. For the Code Object V2 ABI, this will be the first 4 user
// SGPR inputs. We can reserve those and use them directly. // SGPR inputs. We can reserve those and use them directly.
unsigned PrivateSegmentBufferReg = Info.getPreloadedReg( unsigned PrivateSegmentBufferReg = Info.getPreloadedReg(
AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER); AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 LinesSDValue SITargetLowering::LowerFormalArguments(
const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
MachineFunction &MF = DAG.getMachineFunction(); MachineFunction &MF = DAG.getMachineFunction();
const Function &Fn = MF.getFunction(); const Function &Fn = MF.getFunction();
FunctionType *FType = MF.getFunction().getFunctionType(); FunctionType *FType = MF.getFunction().getFunctionType();
SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (Subtarget->isAmdHsaOS() && AMDGPU::isShader(CallConv)) { if (Subtarget->isAmdHsaOS() && AMDGPU::isShader(CallConv)) {
DiagnosticInfoUnsupported NoGraphicsHSA( DiagnosticInfoUnsupported NoGraphicsHSA(
Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()); Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc());
DAG.getContext()->diagnose(NoGraphicsHSA); DAG.getContext()->diagnose(NoGraphicsHSA);
return DAG.getEntryNode(); return DAG.getEntryNode();
} }
▲ Show 20 Lines • Show All 115 Lines • ▼ Show 20 Lines if (IsEntryFunc && VA.isMemLoc()) {
// The first 36 bytes of the input buffer contains information about // The first 36 bytes of the input buffer contains information about
// thread group and global sizes for clover. // thread group and global sizes for clover.
SDValue Arg = lowerKernargMemParameter( SDValue Arg = lowerKernargMemParameter(
DAG, VT, MemVT, DL, Chain, Offset, Align, Ins[i].Flags.isSExt(), &Ins[i]); DAG, VT, MemVT, DL, Chain, Offset, Align, Ins[i].Flags.isSExt(), &Ins[i]);
Chains.push_back(Arg.getValue(1)); Chains.push_back(Arg.getValue(1));
auto *ParamTy = auto *ParamTy =
dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex())); dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
if (Subtarget->getGeneration() == SISubtarget::SOUTHERN_ISLANDS && if (Subtarget->getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS &&
ParamTy && ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) { ParamTy && ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
// On SI local pointers are just offsets into LDS, so they are always // On SI local pointers are just offsets into LDS, so they are always
// less than 16-bits. On CI and newer they could potentially be // less than 16-bits. On CI and newer they could potentially be
// real pointers, so we can't guarantee their size. // real pointers, so we can't guarantee their size.
Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg, Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg,
DAG.getValueType(MVT::i16)); DAG.getValueType(MVT::i16));
} }
▲ Show 20 Lines • Show All 843 Lines • ▼ Show 20 Lines unsigned Reg = StringSwitch<unsigned>(RegName)
.Default(AMDGPU::NoRegister); .Default(AMDGPU::NoRegister);
if (Reg == AMDGPU::NoRegister) { if (Reg == AMDGPU::NoRegister) {
report_fatal_error(Twine("invalid register name \"" report_fatal_error(Twine("invalid register name \""
+ StringRef(RegName) + "\".")); + StringRef(RegName) + "\"."));
} }
if (Subtarget->getGeneration() == SISubtarget::SOUTHERN_ISLANDS && if (Subtarget->getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS &&
Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) { Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) {
report_fatal_error(Twine("invalid register \"" report_fatal_error(Twine("invalid register \""
+ StringRef(RegName) + "\" for subtarget.")); + StringRef(RegName) + "\" for subtarget."));
} }
switch (Reg) { switch (Reg) {
case AMDGPU::M0: case AMDGPU::M0:
case AMDGPU::EXEC_LO: case AMDGPU::EXEC_LO:
▲ Show 20 Lines • Show All 274 Lines • ▼ Show 20 Linesstatic bool setM0ToIndexFromSGPR(const SIInstrInfo *TII,
} }
return true; return true;
} }
// Control flow needs to be inserted if indexing with a VGPR. // Control flow needs to be inserted if indexing with a VGPR.
static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI, static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
MachineBasicBlock &MBB, MachineBasicBlock &MBB,
const SISubtarget &ST) { const GCNSubtarget &ST) {
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo &TRI = TII->getRegisterInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo();
MachineFunction *MF = MBB.getParent(); MachineFunction *MF = MBB.getParent();
MachineRegisterInfo &MRI = MF->getRegInfo(); MachineRegisterInfo &MRI = MF->getRegInfo();
unsigned Dst = MI.getOperand(0).getReg(); unsigned Dst = MI.getOperand(0).getReg();
unsigned SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg(); unsigned SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm(); int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines case 512: // 64 bytes
return AMDGPU::V_MOVRELD_B32_V16; return AMDGPU::V_MOVRELD_B32_V16;
default: default:
llvm_unreachable("unsupported size for MOVRELD pseudos"); llvm_unreachable("unsupported size for MOVRELD pseudos");
} }
} }
static MachineBasicBlock *emitIndirectDst(MachineInstr &MI, static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
MachineBasicBlock &MBB, MachineBasicBlock &MBB,
const SISubtarget &ST) { const GCNSubtarget &ST) {
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo &TRI = TII->getRegisterInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo();
MachineFunction *MF = MBB.getParent(); MachineFunction *MF = MBB.getParent();
MachineRegisterInfo &MRI = MF->getRegInfo(); MachineRegisterInfo &MRI = MF->getRegInfo();
unsigned Dst = MI.getOperand(0).getReg(); unsigned Dst = MI.getOperand(0).getReg();
const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src); const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx); const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
▲ Show 20 Lines • Show All 897 Lines • ▼ Show 20 LinesSDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc);
} }
SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const { SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op); SDLoc SL(Op);
SDValue Chain = Op.getOperand(0); SDValue Chain = Op.getOperand(0);
if (Subtarget->getTrapHandlerAbi() != SISubtarget::TrapHandlerAbiHsa || if (Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
!Subtarget->isTrapHandlerEnabled()) !Subtarget->isTrapHandlerEnabled())
return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain); return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain);
MachineFunction &MF = DAG.getMachineFunction(); MachineFunction &MF = DAG.getMachineFunction();
SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
unsigned UserSGPR = Info->getQueuePtrUserSGPR(); unsigned UserSGPR = Info->getQueuePtrUserSGPR();
assert(UserSGPR != AMDGPU::NoRegister); assert(UserSGPR != AMDGPU::NoRegister);
SDValue QueuePtr = CreateLiveInRegister( SDValue QueuePtr = CreateLiveInRegister(
DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64);
SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01, SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
QueuePtr, SDValue()); QueuePtr, SDValue());
SDValue Ops[] = { SDValue Ops[] = {
ToReg, ToReg,
DAG.getTargetConstant(SISubtarget::TrapIDLLVMTrap, SL, MVT::i16), DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMTrap, SL, MVT::i16),
SGPR01, SGPR01,
ToReg.getValue(1) ToReg.getValue(1)
}; };
return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
} }
SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const { SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op); SDLoc SL(Op);
SDValue Chain = Op.getOperand(0); SDValue Chain = Op.getOperand(0);
MachineFunction &MF = DAG.getMachineFunction(); MachineFunction &MF = DAG.getMachineFunction();
if (Subtarget->getTrapHandlerAbi() != SISubtarget::TrapHandlerAbiHsa || if (Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
!Subtarget->isTrapHandlerEnabled()) { !Subtarget->isTrapHandlerEnabled()) {
DiagnosticInfoUnsupported NoTrap(MF.getFunction(), DiagnosticInfoUnsupported NoTrap(MF.getFunction(),
"debugtrap handler not supported", "debugtrap handler not supported",
Op.getDebugLoc(), Op.getDebugLoc(),
DS_Warning); DS_Warning);
LLVMContext &Ctx = MF.getFunction().getContext(); LLVMContext &Ctx = MF.getFunction().getContext();
Ctx.diagnose(NoTrap); Ctx.diagnose(NoTrap);
return Chain; return Chain;
} }
SDValue Ops[] = { SDValue Ops[] = {
Chain, Chain,
DAG.getTargetConstant(SISubtarget::TrapIDLLVMDebugTrap, SL, MVT::i16) DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMDebugTrap, SL, MVT::i16)
}; };
return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
} }
SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL, SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
SelectionDAG &DAG) const { SelectionDAG &DAG) const {
// FIXME: Use inline constants (src_{shared, private}_base) instead. // FIXME: Use inline constants (src_{shared, private}_base) instead.
if (Subtarget->hasApertureRegs()) { if (Subtarget->hasApertureRegs()) {
▲ Show 20 Lines • Show All 492 Lines • ▼ Show 20 LinesSDValue SITargetLowering::lowerImage(SDValue Op,
} else { } else {
unsigned DMaskIdx; unsigned DMaskIdx;
if (BaseOpcode->Store) { if (BaseOpcode->Store) {
VData = Op.getOperand(2); VData = Op.getOperand(2);
MVT StoreVT = VData.getSimpleValueType(); MVT StoreVT = VData.getSimpleValueType();
if (StoreVT.getScalarType() == MVT::f16) { if (StoreVT.getScalarType() == MVT::f16) {
if (Subtarget->getGeneration() < SISubtarget::VOLCANIC_ISLANDS || if (Subtarget->getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS ||
!BaseOpcode->HasD16) !BaseOpcode->HasD16)
return Op; // D16 is unsupported for this instruction return Op; // D16 is unsupported for this instruction
IsD16 = true; IsD16 = true;
VData = handleD16VData(VData, DAG); VData = handleD16VData(VData, DAG);
} }
NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32; NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
DMaskIdx = 3; DMaskIdx = 3;
} else { } else {
MVT LoadVT = Op.getSimpleValueType(); MVT LoadVT = Op.getSimpleValueType();
if (LoadVT.getScalarType() == MVT::f16) { if (LoadVT.getScalarType() == MVT::f16) {
if (Subtarget->getGeneration() < SISubtarget::VOLCANIC_ISLANDS || if (Subtarget->getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS ||
!BaseOpcode->HasD16) !BaseOpcode->HasD16)
return Op; // D16 is unsupported for this instruction return Op; // D16 is unsupported for this instruction
IsD16 = true; IsD16 = true;
if (LoadVT.isVector() && Subtarget->hasUnpackedD16VMem()) if (LoadVT.isVector() && Subtarget->hasUnpackedD16VMem())
ResultTypes[0] = (LoadVT == MVT::v2f16) ? MVT::v2i32 : MVT::v4i32; ResultTypes[0] = (LoadVT == MVT::v2f16) ? MVT::v2i32 : MVT::v4i32;
} }
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 LinesSDValue SITargetLowering::lowerImage(SDValue Op,
if (BaseOpcode->HasD16) if (BaseOpcode->HasD16)
Ops.push_back(IsD16 ? True : False); Ops.push_back(IsD16 ? True : False);
if (isa<MemSDNode>(Op)) if (isa<MemSDNode>(Op))
Ops.push_back(Op.getOperand(0)); // chain Ops.push_back(Op.getOperand(0)); // chain
int NumVAddrDwords = VAddr.getValueType().getSizeInBits() / 32; int NumVAddrDwords = VAddr.getValueType().getSizeInBits() / 32;
int Opcode = -1; int Opcode = -1;
if (Subtarget->getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) if (Subtarget->getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS)
Opcode = AMDGPU::getMIMGOpcode(Intr->BaseOpcode, AMDGPU::MIMGEncGfx8, Opcode = AMDGPU::getMIMGOpcode(Intr->BaseOpcode, AMDGPU::MIMGEncGfx8,
NumVDataDwords, NumVAddrDwords); NumVDataDwords, NumVAddrDwords);
if (Opcode == -1) if (Opcode == -1)
Opcode = AMDGPU::getMIMGOpcode(Intr->BaseOpcode, AMDGPU::MIMGEncGfx6, Opcode = AMDGPU::getMIMGOpcode(Intr->BaseOpcode, AMDGPU::MIMGEncGfx6,
NumVDataDwords, NumVAddrDwords); NumVDataDwords, NumVAddrDwords);
assert(Opcode != -1); assert(Opcode != -1);
MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops); MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops);
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 LinesSDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::amdgcn_dispatch_id: { case Intrinsic::amdgcn_dispatch_id: {
return getPreloadedValue(DAG, *MFI, VT, AMDGPUFunctionArgInfo::DISPATCH_ID); return getPreloadedValue(DAG, *MFI, VT, AMDGPUFunctionArgInfo::DISPATCH_ID);
} }
case Intrinsic::amdgcn_rcp: case Intrinsic::amdgcn_rcp:
return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_rsq: case Intrinsic::amdgcn_rsq:
return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_rsq_legacy: case Intrinsic::amdgcn_rsq_legacy:
if (Subtarget->getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) if (Subtarget->getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS)
return emitRemovedIntrinsicError(DAG, DL, VT); return emitRemovedIntrinsicError(DAG, DL, VT);
return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_rcp_legacy: case Intrinsic::amdgcn_rcp_legacy:
if (Subtarget->getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) if (Subtarget->getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS)
return emitRemovedIntrinsicError(DAG, DL, VT); return emitRemovedIntrinsicError(DAG, DL, VT);
return DAG.getNode(AMDGPUISD::RCP_LEGACY, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::RCP_LEGACY, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_rsq_clamp: { case Intrinsic::amdgcn_rsq_clamp: {
if (Subtarget->getGeneration() < SISubtarget::VOLCANIC_ISLANDS) if (Subtarget->getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS)
return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1));
Type *Type = VT.getTypeForEVT(*DAG.getContext()); Type *Type = VT.getTypeForEVT(*DAG.getContext());
APFloat Max = APFloat::getLargest(Type->getFltSemantics()); APFloat Max = APFloat::getLargest(Type->getFltSemantics());
APFloat Min = APFloat::getLargest(Type->getFltSemantics(), true); APFloat Min = APFloat::getLargest(Type->getFltSemantics(), true);
SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1)); SDValue Rsq = DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq, SDValue Tmp = DAG.getNode(ISD::FMINNUM, DL, VT, Rsq,
▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 LinesSDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
} }
case Intrinsic::amdgcn_sin: case Intrinsic::amdgcn_sin:
return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_cos: case Intrinsic::amdgcn_cos:
return DAG.getNode(AMDGPUISD::COS_HW, DL, VT, Op.getOperand(1)); return DAG.getNode(AMDGPUISD::COS_HW, DL, VT, Op.getOperand(1));
case Intrinsic::amdgcn_log_clamp: { case Intrinsic::amdgcn_log_clamp: {
if (Subtarget->getGeneration() < SISubtarget::VOLCANIC_ISLANDS) if (Subtarget->getGeneration() < GCNSubtarget::VOLCANIC_ISLANDS)
return SDValue(); return SDValue();
DiagnosticInfoUnsupported BadIntrin( DiagnosticInfoUnsupported BadIntrin(
MF.getFunction(), "intrinsic not supported on subtarget", MF.getFunction(), "intrinsic not supported on subtarget",
DL.getDebugLoc()); DL.getDebugLoc());
DAG.getContext()->diagnose(BadIntrin); DAG.getContext()->diagnose(BadIntrin);
return DAG.getUNDEF(VT); return DAG.getUNDEF(VT);
} }
▲ Show 20 Lines • Show All 416 Lines • ▼ Show 20 Lines if (const ConstantFPSDNode *K = dyn_cast<ConstantFPSDNode>(Src)) {
return DAG.getNode(AMDGPUISD::KILL, DL, MVT::Other, Chain, NegOne); return DAG.getNode(AMDGPUISD::KILL, DL, MVT::Other, Chain, NegOne);
} }
SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Src); SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Src);
return DAG.getNode(AMDGPUISD::KILL, DL, MVT::Other, Chain, Cast); return DAG.getNode(AMDGPUISD::KILL, DL, MVT::Other, Chain, Cast);
} }
case Intrinsic::amdgcn_s_barrier: { case Intrinsic::amdgcn_s_barrier: {
if (getTargetMachine().getOptLevel() > CodeGenOpt::None) { if (getTargetMachine().getOptLevel() > CodeGenOpt::None) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
unsigned WGSize = ST.getFlatWorkGroupSizes(MF.getFunction()).second; unsigned WGSize = ST.getFlatWorkGroupSizes(MF.getFunction()).second;
if (WGSize <= ST.getWavefrontSize()) if (WGSize <= ST.getWavefrontSize())
return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other, return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other,
Op.getOperand(0)), 0); Op.getOperand(0)), 0);
} }
return SDValue(); return SDValue();
}; };
case AMDGPUIntrinsic::SI_tbuffer_store: { case AMDGPUIntrinsic::SI_tbuffer_store: {
▲ Show 20 Lines • Show All 594 Lines • ▼ Show 20 LinesSDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1);
SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3);
SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64,
NegDivScale0, Mul, DivScale1); NegDivScale0, Mul, DivScale1);
SDValue Scale; SDValue Scale;
if (Subtarget->getGeneration() == SISubtarget::SOUTHERN_ISLANDS) { if (Subtarget->getGeneration() == GCNSubtarget::SOUTHERN_ISLANDS) {
// Workaround a hardware bug on SI where the condition output from div_scale // Workaround a hardware bug on SI where the condition output from div_scale
// is not usable. // is not usable.
const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); const SDValue Hi = DAG.getConstant(1, SL, MVT::i32);
// Figure out if the scale to use for div_fmas. // Figure out if the scale to use for div_fmas.
SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y);
▲ Show 20 Lines • Show All 803 Lines • ▼ Show 20 Lines
static bool isKnownNeverSNan(SelectionDAG &DAG, SDValue Op) { static bool isKnownNeverSNan(SelectionDAG &DAG, SDValue Op) {
if (!DAG.getTargetLoweringInfo().hasFloatingPointExceptions()) if (!DAG.getTargetLoweringInfo().hasFloatingPointExceptions())
return true; return true;
return DAG.isKnownNeverNaN(Op); return DAG.isKnownNeverNaN(Op);
} }
static bool isCanonicalized(SelectionDAG &DAG, SDValue Op, static bool isCanonicalized(SelectionDAG &DAG, SDValue Op,
const SISubtarget *ST, unsigned MaxDepth=5) { const GCNSubtarget *ST, unsigned MaxDepth=5) {
// If source is a result of another standard FP operation it is already in // If source is a result of another standard FP operation it is already in
// canonical form. // canonical form.
switch (Op.getOpcode()) { switch (Op.getOpcode()) {
default: default:
break; break;
// These will flush denorms if required. // These will flush denorms if required.
▲ Show 20 Lines • Show All 1,570 Lines • ▼ Show 20 Lines case ISD::CopyFromReg:
R = dyn_cast<RegisterSDNode>(N); R = dyn_cast<RegisterSDNode>(N);
} }
else { else {
R = dyn_cast<RegisterSDNode>(N->getOperand(1)); R = dyn_cast<RegisterSDNode>(N->getOperand(1));
} }
if (R) if (R)
{ {
const MachineFunction * MF = FLI->MF; const MachineFunction * MF = FLI->MF;
const SISubtarget &ST = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
const MachineRegisterInfo &MRI = MF->getRegInfo(); const MachineRegisterInfo &MRI = MF->getRegInfo();
const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo(); const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
unsigned Reg = R->getReg(); unsigned Reg = R->getReg();
if (TRI.isPhysicalRegister(Reg)) if (TRI.isPhysicalRegister(Reg))
return TRI.isVGPR(MRI, Reg); return TRI.isVGPR(MRI, Reg);
if (MRI.isLiveIn(Reg)) { if (MRI.isLiveIn(Reg)) {
// workitem.id.x workitem.id.y workitem.id.z // workitem.id.x workitem.id.y workitem.id.z
Show All 39 Lines

lib/Target/AMDGPU/SIInsertSkips.cpp

Show First 20 Lines • Show All 333 Lines • ▼ Show 20 Linesbool SIInsertSkips::skipMaskBranch(MachineInstr &MI,
BuildMI(SrcMBB, InsPt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) BuildMI(SrcMBB, InsPt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
.addMBB(DestBB); .addMBB(DestBB);
return true; return true;
} }
bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) { bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo(); TII = ST.getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
SkipThreshold = SkipThresholdFlag; SkipThreshold = SkipThresholdFlag;
bool HaveKill = false; bool HaveKill = false;
bool MadeChange = false; bool MadeChange = false;
// Track depth of exec mask, divergent branches. // Track depth of exec mask, divergent branches.
▲ Show 20 Lines • Show All 95 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIInsertWaitcnts.cpp

Show First 20 Lines • Show All 130 Lines • ▼ Show 20 Lines
// of each wait counter, and a per-register scoreboard for each wait counter. // of each wait counter, and a per-register scoreboard for each wait counter.
// We also maintain the latest score for every event type that can change the // We also maintain the latest score for every event type that can change the
// waitcnt in order to know if there are multiple types of events within // waitcnt in order to know if there are multiple types of events within
// the brackets. When multiple types of event happen in the bracket, // the brackets. When multiple types of event happen in the bracket,
// wait count may get decreased out of order, therefore we need to put in // wait count may get decreased out of order, therefore we need to put in
// "s_waitcnt 0" before use. // "s_waitcnt 0" before use.
class BlockWaitcntBrackets { class BlockWaitcntBrackets {
public: public:
BlockWaitcntBrackets(const SISubtarget *SubTarget) : ST(SubTarget) { BlockWaitcntBrackets(const GCNSubtarget *SubTarget) : ST(SubTarget) {
for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS; for (enum InstCounterType T = VM_CNT; T < NUM_INST_CNTS;
T = (enum InstCounterType)(T + 1)) { T = (enum InstCounterType)(T + 1)) {
memset(VgprScores[T], 0, sizeof(VgprScores[T])); memset(VgprScores[T], 0, sizeof(VgprScores[T]));
} }
} }
~BlockWaitcntBrackets() = default; ~BlockWaitcntBrackets() = default;
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Linespublic:
void setMixedExpTypes(bool MixedExpTypesIn) { void setMixedExpTypes(bool MixedExpTypesIn) {
MixedExpTypes = MixedExpTypesIn; MixedExpTypes = MixedExpTypesIn;
} }
void print(raw_ostream &); void print(raw_ostream &);
void dump() { print(dbgs()); } void dump() { print(dbgs()); }
private: private:
const SISubtarget *ST = nullptr; const GCNSubtarget *ST = nullptr;
bool WaitAtBeginning = false; bool WaitAtBeginning = false;
bool RevisitLoop = false; bool RevisitLoop = false;
bool MixedExpTypes = false; bool MixedExpTypes = false;
int32_t PostOrder = 0; int32_t PostOrder = 0;
MachineInstr *Waitcnt = nullptr; MachineInstr *Waitcnt = nullptr;
int32_t ScoreLBs[NUM_INST_CNTS] = {0}; int32_t ScoreLBs[NUM_INST_CNTS] = {0};
int32_t ScoreUBs[NUM_INST_CNTS] = {0}; int32_t ScoreUBs[NUM_INST_CNTS] = {0};
int32_t EventUBs[NUM_WAIT_EVENTS] = {0}; int32_t EventUBs[NUM_WAIT_EVENTS] = {0};
Show All 33 Linesprivate:
MachineInstr *LfWaitcnt = nullptr; MachineInstr *LfWaitcnt = nullptr;
// Number of iterations the loop has been visited, not including the initial // Number of iterations the loop has been visited, not including the initial
// walk over. // walk over.
int32_t IterCnt = 0; int32_t IterCnt = 0;
}; };
class SIInsertWaitcnts : public MachineFunctionPass { class SIInsertWaitcnts : public MachineFunctionPass {
private: private:
const SISubtarget *ST = nullptr; const GCNSubtarget *ST = nullptr;
const SIInstrInfo *TII = nullptr; const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr; const SIRegisterInfo *TRI = nullptr;
const MachineRegisterInfo *MRI = nullptr; const MachineRegisterInfo *MRI = nullptr;
const MachineLoopInfo *MLI = nullptr; const MachineLoopInfo *MLI = nullptr;
AMDGPU::IsaInfo::IsaVersion IV; AMDGPU::IsaInfo::IsaVersion IV;
AMDGPUAS AMDGPUASI; AMDGPUAS AMDGPUASI;
DenseSet<MachineBasicBlock *> BlockVisitedSet; DenseSet<MachineBasicBlock *> BlockVisitedSet;
▲ Show 20 Lines • Show All 749 Lines • ▼ Show 20 Lines EmitWaitcnt |= ScoreBrackets->updateByWait(
EXP_CNT, ScoreBrackets->getScoreUB(EXP_CNT)); EXP_CNT, ScoreBrackets->getScoreUB(EXP_CNT));
EmitWaitcnt |= ScoreBrackets->updateByWait( EmitWaitcnt |= ScoreBrackets->updateByWait(
LGKM_CNT, ScoreBrackets->getScoreUB(LGKM_CNT)); LGKM_CNT, ScoreBrackets->getScoreUB(LGKM_CNT));
} }
// TODO: Remove this work-around, enable the assert for Bug 457939 // TODO: Remove this work-around, enable the assert for Bug 457939
// after fixing the scheduler. Also, the Shader Compiler code is // after fixing the scheduler. Also, the Shader Compiler code is
// independent of target. // independent of target.
if (readsVCCZ(MI) && ST->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) { if (readsVCCZ(MI) && ST->getGeneration() <= GCNSubtarget::SEA_ISLANDS) {
if (ScoreBrackets->getScoreLB(LGKM_CNT) < if (ScoreBrackets->getScoreLB(LGKM_CNT) <
ScoreBrackets->getScoreUB(LGKM_CNT) && ScoreBrackets->getScoreUB(LGKM_CNT) &&
ScoreBrackets->hasPendingSMEM()) { ScoreBrackets->hasPendingSMEM()) {
// Wait on everything, not just LGKM. vccz reads usually come from // Wait on everything, not just LGKM. vccz reads usually come from
// terminators, and we always wait on everything at the end of the // terminators, and we always wait on everything at the end of the
// block, so if we only wait on LGKM here, we might end up with // block, so if we only wait on LGKM here, we might end up with
// another s_waitcnt inserted right after this if there are non-LGKM // another s_waitcnt inserted right after this if there are non-LGKM
// instructions still outstanding. // instructions still outstanding.
▲ Show 20 Lines • Show All 568 Lines • ▼ Show 20 Lines for (MachineBasicBlock::iterator Iter = Block.begin(), E = Block.end();
} }
bool VCCZBugWorkAround = false; bool VCCZBugWorkAround = false;
if (readsVCCZ(Inst) && if (readsVCCZ(Inst) &&
(!VCCZBugHandledSet.count(&Inst))) { (!VCCZBugHandledSet.count(&Inst))) {
if (ScoreBrackets->getScoreLB(LGKM_CNT) < if (ScoreBrackets->getScoreLB(LGKM_CNT) <
ScoreBrackets->getScoreUB(LGKM_CNT) && ScoreBrackets->getScoreUB(LGKM_CNT) &&
ScoreBrackets->hasPendingSMEM()) { ScoreBrackets->hasPendingSMEM()) {
if (ST->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) if (ST->getGeneration() <= GCNSubtarget::SEA_ISLANDS)
VCCZBugWorkAround = true; VCCZBugWorkAround = true;
} }
} }
// Generate an s_waitcnt instruction to be placed before // Generate an s_waitcnt instruction to be placed before
// cur_Inst, if needed. // cur_Inst, if needed.
generateWaitcntInstBefore(Inst, ScoreBrackets); generateWaitcntInstBefore(Inst, ScoreBrackets);
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines#endif
insertWaitcntBeforeCF(Block, SWaitInst); insertWaitcntBeforeCF(Block, SWaitInst);
WaitcntData->setWaitcnt(SWaitInst); WaitcntData->setWaitcnt(SWaitInst);
} }
} }
} }
} }
bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) { bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
ST = &MF.getSubtarget<SISubtarget>(); ST = &MF.getSubtarget<GCNSubtarget>();
TII = ST->getInstrInfo(); TII = ST->getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
MLI = &getAnalysis<MachineLoopInfo>(); MLI = &getAnalysis<MachineLoopInfo>();
IV = AMDGPU::IsaInfo::getIsaVersion(ST->getFeatureBits()); IV = AMDGPU::IsaInfo::getIsaVersion(ST->getFeatureBits());
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
AMDGPUASI = ST->getAMDGPUAS(); AMDGPUASI = ST->getAMDGPUAS();
▲ Show 20 Lines • Show All 174 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIInstrFormats.td

//===-- SIInstrFormats.td - SI Instruction Encodings ----------------------===// //===-- SIInstrFormats.td - SI Instruction Encodings ----------------------===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
// This file is distributed under the University of Illinois Open Source // This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details. // License. See LICENSE.TXT for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// //
// SI Instruction format definitions. // SI Instruction format definitions.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
def isGCN : Predicate<"Subtarget->getGeneration() " def isGCN : Predicate<"Subtarget->getGeneration() "
">= SISubtarget::SOUTHERN_ISLANDS">, ">= GCNSubtarget::SOUTHERN_ISLANDS">,
AssemblerPredicate<"FeatureGCN">; AssemblerPredicate<"FeatureGCN">;
def isSI : Predicate<"Subtarget->getGeneration() " def isSI : Predicate<"Subtarget->getGeneration() "
"== SISubtarget::SOUTHERN_ISLANDS">, "== GCNSubtarget::SOUTHERN_ISLANDS">,
AssemblerPredicate<"FeatureSouthernIslands">; AssemblerPredicate<"FeatureSouthernIslands">;
class InstSI <dag outs, dag ins, string asm = "", class InstSI <dag outs, dag ins, string asm = "",
list<dag> pattern = []> : list<dag> pattern = []> :
AMDGPUInst<outs, ins, asm, pattern>, GCNPredicateControl { AMDGPUInst<outs, ins, asm, pattern>, GCNPredicateControl {
let SubtargetPredicate = isGCN; let SubtargetPredicate = isGCN;
▲ Show 20 Lines • Show All 310 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIInstrInfo.h

Show All 33 Lines
#define GET_INSTRINFO_HEADER #define GET_INSTRINFO_HEADER
#include "AMDGPUGenInstrInfo.inc" #include "AMDGPUGenInstrInfo.inc"
namespace llvm { namespace llvm {
class APInt; class APInt;
class MachineRegisterInfo; class MachineRegisterInfo;
class RegScavenger; class RegScavenger;
class SISubtarget; class GCNSubtarget;
class TargetRegisterClass; class TargetRegisterClass;
class SIInstrInfo final : public AMDGPUGenInstrInfo { class SIInstrInfo final : public AMDGPUGenInstrInfo {
private: private:
const SIRegisterInfo RI; const SIRegisterInfo RI;
const SISubtarget &ST; const GCNSubtarget &ST;
// The inverse predicate should have the negative value. // The inverse predicate should have the negative value.
enum BranchPredicate { enum BranchPredicate {
INVALID_BR = 0, INVALID_BR = 0,
SCC_TRUE = 1, SCC_TRUE = 1,
SCC_FALSE = -1, SCC_FALSE = -1,
VCCNZ = 2, VCCNZ = 2,
VCCZ = -2, VCCZ = -2,
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines enum TargetOperandFlags {
MO_GOTPCREL32_HI = 3, MO_GOTPCREL32_HI = 3,
// MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO. // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
MO_REL32 = 4, MO_REL32 = 4,
MO_REL32_LO = 4, MO_REL32_LO = 4,
// MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI. // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
MO_REL32_HI = 5 MO_REL32_HI = 5
}; };
explicit SIInstrInfo(const SISubtarget &ST); explicit SIInstrInfo(const GCNSubtarget &ST);
const SIRegisterInfo &getRegisterInfo() const { const SIRegisterInfo &getRegisterInfo() const {
return RI; return RI;
} }
bool isReallyTriviallyReMaterializable(const MachineInstr &MI, bool isReallyTriviallyReMaterializable(const MachineInstr &MI,
AliasAnalysis *AA) const override; AliasAnalysis *AA) const override;
▲ Show 20 Lines • Show All 799 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIInstrInfo.cpp

Show First 20 Lines • Show All 78 Lines • ▼ Show 20 Lines
// Must be at least 4 to be able to branch over minimum unconditional branch // Must be at least 4 to be able to branch over minimum unconditional branch
// code. This is only for making it possible to write reasonably small tests for // code. This is only for making it possible to write reasonably small tests for
// long branches. // long branches.
static cl::opt<unsigned> static cl::opt<unsigned>
BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
cl::desc("Restrict range of branch instructions (DEBUG)")); cl::desc("Restrict range of branch instructions (DEBUG)"));
SIInstrInfo::SIInstrInfo(const SISubtarget &ST) SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
: AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
RI(ST), ST(ST) {} RI(ST), ST(ST) {}
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
// TargetInstrInfo callbacks // TargetInstrInfo callbacks
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
static unsigned getNumOperandsNoGlue(SDNode *Node) { static unsigned getNumOperandsNoGlue(SDNode *Node) {
▲ Show 20 Lines • Show All 934 Lines • ▼ Show 20 Lines
} }
/// \param @Offset Offset in bytes of the FrameIndex being spilled /// \param @Offset Offset in bytes of the FrameIndex being spilled
unsigned SIInstrInfo::calculateLDSSpillAddress( unsigned SIInstrInfo::calculateLDSSpillAddress(
MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg, MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
unsigned FrameOffset, unsigned Size) const { unsigned FrameOffset, unsigned Size) const {
MachineFunction *MF = MBB.getParent(); MachineFunction *MF = MBB.getParent();
SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
const AMDGPUSubtarget &ST = MF->getSubtarget<AMDGPUSubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
DebugLoc DL = MBB.findDebugLoc(MI); DebugLoc DL = MBB.findDebugLoc(MI);
unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize(); unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
unsigned WavefrontSize = ST.getWavefrontSize(); unsigned WavefrontSize = ST.getWavefrontSize();
unsigned TIDReg = MFI->getTIDReg(); unsigned TIDReg = MFI->getTIDReg();
if (!MFI->hasCalculatedTID()) { if (!MFI->hasCalculatedTID()) {
MachineBasicBlock &Entry = MBB.getParent()->front(); MachineBasicBlock &Entry = MBB.getParent()->front();
MachineBasicBlock::iterator Insert = Entry.front(); MachineBasicBlock::iterator Insert = Entry.front();
▲ Show 20 Lines • Show All 1,863 Lines • ▼ Show 20 Lines if (MI.mayStore()) {
const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
if (Soff && Soff->getReg() != AMDGPU::M0) { if (Soff && Soff->getReg() != AMDGPU::M0) {
ErrInfo = "scalar stores must use m0 as offset register"; ErrInfo = "scalar stores must use m0 as offset register";
return false; return false;
} }
} }
} }
if (isFLAT(MI) && !MF->getSubtarget<SISubtarget>().hasFlatInstOffsets()) { if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
if (Offset->getImm() != 0) { if (Offset->getImm() != 0) {
ErrInfo = "subtarget does not support offsets in flat instructions"; ErrInfo = "subtarget does not support offsets in flat instructions";
return false; return false;
} }
} }
const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
▲ Show 20 Lines • Show All 734 Lines • ▼ Show 20 Lines if (VAddr) {
BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
.addReg(NewVAddrLo) .addReg(NewVAddrLo)
.addImm(AMDGPU::sub0) .addImm(AMDGPU::sub0)
.addReg(NewVAddrHi) .addReg(NewVAddrHi)
.addImm(AMDGPU::sub1); .addImm(AMDGPU::sub1);
} else { } else {
// This instructions is the _OFFSET variant, so we need to convert it to // This instructions is the _OFFSET variant, so we need to convert it to
// ADDR64. // ADDR64.
assert(MBB.getParent()->getSubtarget<SISubtarget>().getGeneration() assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
< SISubtarget::VOLCANIC_ISLANDS && < GCNSubtarget::VOLCANIC_ISLANDS &&
"FIXME: Need to emit flat atomics here"); "FIXME: Need to emit flat atomics here");
MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
// Atomics rith return have have an additional tied operand and are // Atomics rith return have have an additional tied operand and are
▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines case AMDGPU::S_BCNT1_I32_B64:
continue; continue;
case AMDGPU::S_BFE_I64: case AMDGPU::S_BFE_I64:
splitScalar64BitBFE(Worklist, Inst); splitScalar64BitBFE(Worklist, Inst);
Inst.eraseFromParent(); Inst.eraseFromParent();
continue; continue;
case AMDGPU::S_LSHL_B32: case AMDGPU::S_LSHL_B32:
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
NewOpcode = AMDGPU::V_LSHLREV_B32_e64; NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
swapOperands(Inst); swapOperands(Inst);
} }
break; break;
case AMDGPU::S_ASHR_I32: case AMDGPU::S_ASHR_I32:
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
NewOpcode = AMDGPU::V_ASHRREV_I32_e64; NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
swapOperands(Inst); swapOperands(Inst);
} }
break; break;
case AMDGPU::S_LSHR_B32: case AMDGPU::S_LSHR_B32:
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
NewOpcode = AMDGPU::V_LSHRREV_B32_e64; NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
swapOperands(Inst); swapOperands(Inst);
} }
break; break;
case AMDGPU::S_LSHL_B64: case AMDGPU::S_LSHL_B64:
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
NewOpcode = AMDGPU::V_LSHLREV_B64; NewOpcode = AMDGPU::V_LSHLREV_B64;
swapOperands(Inst); swapOperands(Inst);
} }
break; break;
case AMDGPU::S_ASHR_I64: case AMDGPU::S_ASHR_I64:
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
NewOpcode = AMDGPU::V_ASHRREV_I64; NewOpcode = AMDGPU::V_ASHRREV_I64;
swapOperands(Inst); swapOperands(Inst);
} }
break; break;
case AMDGPU::S_LSHR_B64: case AMDGPU::S_LSHR_B64:
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS) {
NewOpcode = AMDGPU::V_LSHRREV_B64; NewOpcode = AMDGPU::V_LSHRREV_B64;
swapOperands(Inst); swapOperands(Inst);
} }
break; break;
case AMDGPU::S_ABS_I32: case AMDGPU::S_ABS_I32:
lowerScalarAbs(Worklist, Inst); lowerScalarAbs(Worklist, Inst);
Inst.eraseFromParent(); Inst.eraseFromParent();
▲ Show 20 Lines • Show All 783 Lines • ▼ Show 20 LinesMachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
return &MI.getOperand(Idx); return &MI.getOperand(Idx);
} }
uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
if (ST.isAmdHsaOS()) { if (ST.isAmdHsaOS()) {
// Set ATC = 1. GFX9 doesn't have this bit. // Set ATC = 1. GFX9 doesn't have this bit.
if (ST.getGeneration() <= SISubtarget::VOLCANIC_ISLANDS) if (ST.getGeneration() <= GCNSubtarget::VOLCANIC_ISLANDS)
RsrcDataFormat |= (1ULL << 56); RsrcDataFormat |= (1ULL << 56);
// Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
// BTW, it disables TC L2 and therefore decreases performance. // BTW, it disables TC L2 and therefore decreases performance.
if (ST.getGeneration() == SISubtarget::VOLCANIC_ISLANDS) if (ST.getGeneration() == GCNSubtarget::VOLCANIC_ISLANDS)
RsrcDataFormat |= (2ULL << 59); RsrcDataFormat |= (2ULL << 59);
} }
return RsrcDataFormat; return RsrcDataFormat;
} }
uint64_t SIInstrInfo::getScratchRsrcWords23() const { uint64_t SIInstrInfo::getScratchRsrcWords23() const {
uint64_t Rsrc23 = getDefaultRsrcDataFormat() | uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
AMDGPU::RSRC_TID_ENABLE | AMDGPU::RSRC_TID_ENABLE |
0xffffffff; // Size; 0xffffffff; // Size;
// GFX9 doesn't have ELEMENT_SIZE. // GFX9 doesn't have ELEMENT_SIZE.
if (ST.getGeneration() <= SISubtarget::VOLCANIC_ISLANDS) { if (ST.getGeneration() <= GCNSubtarget::VOLCANIC_ISLANDS) {
uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1; uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
} }
// IndexStride = 64. // IndexStride = 64.
Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
// If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
// Clear them unless we want a huge stride. // Clear them unless we want a huge stride.
if (ST.getGeneration() >= SISubtarget::VOLCANIC_ISLANDS) if (ST.getGeneration() >= GCNSubtarget::VOLCANIC_ISLANDS)
Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
return Rsrc23; return Rsrc23;
} }
bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
unsigned Opc = MI.getOpcode(); unsigned Opc = MI.getOpcode();
▲ Show 20 Lines • Show All 318 Lines • ▼ Show 20 Linesenum SIEncodingFamily {
SI = 0, SI = 0,
VI = 1, VI = 1,
SDWA = 2, SDWA = 2,
SDWA9 = 3, SDWA9 = 3,
GFX80 = 4, GFX80 = 4,
GFX9 = 5 GFX9 = 5
}; };
static SIEncodingFamily subtargetEncodingFamily(const SISubtarget &ST) { static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) {
switch (ST.getGeneration()) { switch (ST.getGeneration()) {
case SISubtarget::SOUTHERN_ISLANDS: case GCNSubtarget::SOUTHERN_ISLANDS:
case SISubtarget::SEA_ISLANDS: case GCNSubtarget::SEA_ISLANDS:
return SIEncodingFamily::SI; return SIEncodingFamily::SI;
case SISubtarget::VOLCANIC_ISLANDS: case GCNSubtarget::VOLCANIC_ISLANDS:
case SISubtarget::GFX9: case GCNSubtarget::GFX9:
return SIEncodingFamily::VI; return SIEncodingFamily::VI;
} }
llvm_unreachable("Unknown subtarget generation!"); llvm_unreachable("Unknown subtarget generation!");
} }
int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
SIEncodingFamily Gen = subtargetEncodingFamily(ST); SIEncodingFamily Gen = subtargetEncodingFamily(ST);
if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
ST.getGeneration() >= SISubtarget::GFX9) ST.getGeneration() >= GCNSubtarget::GFX9)
Gen = SIEncodingFamily::GFX9; Gen = SIEncodingFamily::GFX9;
if (get(Opcode).TSFlags & SIInstrFlags::SDWA) if (get(Opcode).TSFlags & SIInstrFlags::SDWA)
Gen = ST.getGeneration() == SISubtarget::GFX9 ? SIEncodingFamily::SDWA9 Gen = ST.getGeneration() == GCNSubtarget::GFX9 ? SIEncodingFamily::SDWA9
: SIEncodingFamily::SDWA; : SIEncodingFamily::SDWA;
// Adjust the encoding family to GFX80 for D16 buffer instructions when the // Adjust the encoding family to GFX80 for D16 buffer instructions when the
// subtarget has UnpackedD16VMem feature. // subtarget has UnpackedD16VMem feature.
// TODO: remove this when we discard GFX80 encoding. // TODO: remove this when we discard GFX80 encoding.
if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))
Gen = SIEncodingFamily::GFX80; Gen = SIEncodingFamily::GFX80;
int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
Show All 12 Lines

lib/Target/AMDGPU/SIInstrInfo.td

//===-- SIInstrInfo.td - SI Instruction Infos -------------*- tablegen -*--===// //===-- SIInstrInfo.td - SI Instruction Infos -------------*- tablegen -*--===//
// //
// The LLVM Compiler Infrastructure // The LLVM Compiler Infrastructure
// //
// This file is distributed under the University of Illinois Open Source // This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details. // License. See LICENSE.TXT for details.
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
def isCI : Predicate<"Subtarget->getGeneration() " def isCI : Predicate<"Subtarget->getGeneration() "
">= SISubtarget::SEA_ISLANDS">; ">= GCNSubtarget::SEA_ISLANDS">;
def isCIOnly : Predicate<"Subtarget->getGeneration() ==" def isCIOnly : Predicate<"Subtarget->getGeneration() =="
"SISubtarget::SEA_ISLANDS">, "GCNSubtarget::SEA_ISLANDS">,
AssemblerPredicate <"FeatureSeaIslands">; AssemblerPredicate <"FeatureSeaIslands">;
def isVIOnly : Predicate<"Subtarget->getGeneration() ==" def isVIOnly : Predicate<"Subtarget->getGeneration() =="
"SISubtarget::VOLCANIC_ISLANDS">, "GCNSubtarget::VOLCANIC_ISLANDS">,
AssemblerPredicate <"FeatureVolcanicIslands">; AssemblerPredicate <"FeatureVolcanicIslands">;
def DisableInst : Predicate <"false">, AssemblerPredicate<"FeatureDisable">; def DisableInst : Predicate <"false">, AssemblerPredicate<"FeatureDisable">;
class GCNPredicateControl : PredicateControl { class GCNPredicateControl : PredicateControl {
Predicate SIAssemblerPredicate = isSICI; Predicate SIAssemblerPredicate = isSICI;
Predicate VIAssemblerPredicate = isVI; Predicate VIAssemblerPredicate = isVI;
} }
▲ Show 20 Lines • Show All 458 Lines • ▼ Show 20 Linesclass InlineImm <ValueType vt> : PatLeaf <(vt imm), [{
return isInlineImmediate(N); return isInlineImmediate(N);
}]>; }]>;
class InlineFPImm <ValueType vt> : PatLeaf <(vt fpimm), [{ class InlineFPImm <ValueType vt> : PatLeaf <(vt fpimm), [{
return isInlineImmediate(N); return isInlineImmediate(N);
}]>; }]>;
class VGPRImm <dag frag> : PatLeaf<frag, [{ class VGPRImm <dag frag> : PatLeaf<frag, [{
if (Subtarget->getGeneration() < SISubtarget::SOUTHERN_ISLANDS) { if (Subtarget->getGeneration() < GCNSubtarget::SOUTHERN_ISLANDS) {
return false; return false;
} }
const SIRegisterInfo *SIRI = const SIRegisterInfo *SIRI =
static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
unsigned Limit = 0; unsigned Limit = 0;
for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end(); for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
Limit < 10 && U != E; ++U, ++Limit) { Limit < 10 && U != E; ++U, ++Limit) {
const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo()); const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
▲ Show 20 Lines • Show All 1,511 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SILoadStoreOptimizer.cpp

Show First 20 Lines • Show All 97 Lines • ▼ Show 20 Lines struct CombineInfo {
bool SLC0; bool SLC0;
bool SLC1; bool SLC1;
bool UseST64; bool UseST64;
bool IsX2; bool IsX2;
SmallVector<MachineInstr*, 8> InstsToMove; SmallVector<MachineInstr*, 8> InstsToMove;
}; };
private: private:
const SISubtarget *STM = nullptr; const GCNSubtarget *STM = nullptr;
const SIInstrInfo *TII = nullptr; const SIInstrInfo *TII = nullptr;
const SIRegisterInfo *TRI = nullptr; const SIRegisterInfo *TRI = nullptr;
MachineRegisterInfo *MRI = nullptr; MachineRegisterInfo *MRI = nullptr;
AliasAnalysis *AA = nullptr; AliasAnalysis *AA = nullptr;
unsigned CreatedX2; unsigned CreatedX2;
static bool offsetsCanBeCombined(CombineInfo &CI); static bool offsetsCanBeCombined(CombineInfo &CI);
▲ Show 20 Lines • Show All 819 Lines • ▼ Show 20 Linesbool SILoadStoreOptimizer::optimizeBlock(MachineBasicBlock &MBB) {
return Modified; return Modified;
} }
bool SILoadStoreOptimizer::runOnMachineFunction(MachineFunction &MF) { bool SILoadStoreOptimizer::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction())) if (skipFunction(MF.getFunction()))
return false; return false;
STM = &MF.getSubtarget<SISubtarget>(); STM = &MF.getSubtarget<GCNSubtarget>();
if (!STM->loadStoreOptEnabled()) if (!STM->loadStoreOptEnabled())
return false; return false;
TII = STM->getInstrInfo(); TII = STM->getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
Show All 18 Lines

lib/Target/AMDGPU/SILowerControlFlow.cpp

Show First 20 Lines • Show All 480 Lines • ▼ Show 20 Linesvoid SILowerControlFlow::combineMasks(MachineInstr &MI) {
unsigned Reg = MI.getOperand(OpToReplace).getReg(); unsigned Reg = MI.getOperand(OpToReplace).getReg();
MI.RemoveOperand(OpToReplace); MI.RemoveOperand(OpToReplace);
MI.addOperand(Ops[UniqueOpndIdx]); MI.addOperand(Ops[UniqueOpndIdx]);
if (MRI->use_empty(Reg)) if (MRI->use_empty(Reg))
MRI->getUniqueVRegDef(Reg)->eraseFromParent(); MRI->getUniqueVRegDef(Reg)->eraseFromParent();
} }
bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) { bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo(); TII = ST.getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
// This doesn't actually need LiveIntervals, but we can preserve them. // This doesn't actually need LiveIntervals, but we can preserve them.
LIS = getAnalysisIfAvailable<LiveIntervals>(); LIS = getAnalysisIfAvailable<LiveIntervals>();
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
MachineFunction::iterator NextBB; MachineFunction::iterator NextBB;
▲ Show 20 Lines • Show All 59 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SILowerI1Copies.cpp

Show First 20 Lines • Show All 60 Lines • ▼ Show 20 Lines
char &llvm::SILowerI1CopiesID = SILowerI1Copies::ID; char &llvm::SILowerI1CopiesID = SILowerI1Copies::ID;
FunctionPass *llvm::createSILowerI1CopiesPass() { FunctionPass *llvm::createSILowerI1CopiesPass() {
return new SILowerI1Copies(); return new SILowerI1Copies();
} }
bool SILowerI1Copies::runOnMachineFunction(MachineFunction &MF) { bool SILowerI1Copies::runOnMachineFunction(MachineFunction &MF) {
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const TargetRegisterInfo *TRI = &TII->getRegisterInfo(); const TargetRegisterInfo *TRI = &TII->getRegisterInfo();
std::vector<unsigned> I1Defs; std::vector<unsigned> I1Defs;
for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
BI != BE; ++BI) { BI != BE; ++BI) {
▲ Show 20 Lines • Show All 90 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Lines : AMDGPUMachineFunction(MF),
PrivateSegmentWaveByteOffset(false), PrivateSegmentWaveByteOffset(false),
WorkItemIDX(false), WorkItemIDX(false),
WorkItemIDY(false), WorkItemIDY(false),
WorkItemIDZ(false), WorkItemIDZ(false),
ImplicitBufferPtr(false), ImplicitBufferPtr(false),
ImplicitArgPtr(false), ImplicitArgPtr(false),
GITPtrHigh(0xffffffff), GITPtrHigh(0xffffffff),
HighBitsOf32BitAddress(0) { HighBitsOf32BitAddress(0) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const Function &F = MF.getFunction(); const Function &F = MF.getFunction();
FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F); FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
WavesPerEU = ST.getWavesPerEU(F); WavesPerEU = ST.getWavesPerEU(F);
Occupancy = getMaxWavesPerEU(); Occupancy = getMaxWavesPerEU();
limitOccupancy(MF); limitOccupancy(MF);
if (!isEntryFunction()) { if (!isEntryFunction()) {
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines if (isEntryFunction()) {
// enabled if Z is. // enabled if Z is.
if (WorkItemIDZ) if (WorkItemIDZ)
WorkItemIDY = true; WorkItemIDY = true;
if (HasStackObjects || MaySpill) { if (HasStackObjects || MaySpill) {
PrivateSegmentWaveByteOffset = true; PrivateSegmentWaveByteOffset = true;
// HS and GS always have the scratch wave offset in SGPR5 on GFX9. // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 && if (ST.getGeneration() >= GCNSubtarget::GFX9 &&
(CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS)) (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
ArgInfo.PrivateSegmentWaveByteOffset ArgInfo.PrivateSegmentWaveByteOffset
= ArgDescriptor::createRegister(AMDGPU::SGPR5); = ArgDescriptor::createRegister(AMDGPU::SGPR5);
} }
} }
bool IsCOV2 = ST.isAmdCodeObjectV2(F); bool IsCOV2 = ST.isAmdCodeObjectV2(F);
if (IsCOV2) { if (IsCOV2) {
Show All 31 LinesSIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF)
A = F.getFnAttribute("amdgpu-32bit-address-high-bits"); A = F.getFnAttribute("amdgpu-32bit-address-high-bits");
S = A.getValueAsString(); S = A.getValueAsString();
if (!S.empty()) if (!S.empty())
S.consumeInteger(0, HighBitsOf32BitAddress); S.consumeInteger(0, HighBitsOf32BitAddress);
} }
void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) { void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
limitOccupancy(getMaxWavesPerEU()); limitOccupancy(getMaxWavesPerEU());
const SISubtarget& ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(), limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(),
MF.getFunction())); MF.getFunction()));
} }
unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer( unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer(
const SIRegisterInfo &TRI) { const SIRegisterInfo &TRI) {
ArgInfo.PrivateSegmentBuffer = ArgInfo.PrivateSegmentBuffer =
ArgDescriptor::createRegister(TRI.getMatchingSuperReg( ArgDescriptor::createRegister(TRI.getMatchingSuperReg(
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines
bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF, bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
int FI) { int FI) {
std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI]; std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI];
// This has already been allocated. // This has already been allocated.
if (!SpillLanes.empty()) if (!SpillLanes.empty())
return true; return true;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo();
MachineFrameInfo &FrameInfo = MF.getFrameInfo(); MachineFrameInfo &FrameInfo = MF.getFrameInfo();
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned WaveSize = ST.getWavefrontSize(); unsigned WaveSize = ST.getWavefrontSize();
unsigned Size = FrameInfo.getObjectSize(FI); unsigned Size = FrameInfo.getObjectSize(FI);
assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size"); assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs"); assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs");
▲ Show 20 Lines • Show All 73 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIMemoryLegalizer.cpp

Show First 20 Lines • Show All 251 Lines • ▼ Show 20 Lines
class SICacheControl { class SICacheControl {
protected: protected:
/// Instruction info. /// Instruction info.
const SIInstrInfo *TII = nullptr; const SIInstrInfo *TII = nullptr;
IsaInfo::IsaVersion IV; IsaInfo::IsaVersion IV;
SICacheControl(const SISubtarget &ST); SICacheControl(const GCNSubtarget &ST);
public: public:
/// Create a cache control for the subtarget \p ST. /// Create a cache control for the subtarget \p ST.
static std::unique_ptr<SICacheControl> create(const SISubtarget &ST); static std::unique_ptr<SICacheControl> create(const GCNSubtarget &ST);
/// Update \p MI memory load instruction to bypass any caches up to /// Update \p MI memory load instruction to bypass any caches up to
/// the \p Scope memory scope for address spaces \p /// the \p Scope memory scope for address spaces \p
/// AddrSpace. Return true iff the instruction was modified. /// AddrSpace. Return true iff the instruction was modified.
virtual bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, virtual bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
SIAtomicScope Scope, SIAtomicScope Scope,
SIAtomicAddrSpace AddrSpace) const = 0; SIAtomicAddrSpace AddrSpace) const = 0;
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Linesprotected:
/// Sets SLC bit to "true" if present in \p MI. Returns true if \p MI /// Sets SLC bit to "true" if present in \p MI. Returns true if \p MI
/// is modified, false otherwise. /// is modified, false otherwise.
bool enableSLCBit(const MachineBasicBlock::iterator &MI) const { bool enableSLCBit(const MachineBasicBlock::iterator &MI) const {
return enableNamedBit<AMDGPU::OpName::slc>(MI); return enableNamedBit<AMDGPU::OpName::slc>(MI);
} }
public: public:
SIGfx6CacheControl(const SISubtarget &ST) : SICacheControl(ST) {}; SIGfx6CacheControl(const GCNSubtarget &ST) : SICacheControl(ST) {};
bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI, bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
SIAtomicScope Scope, SIAtomicScope Scope,
SIAtomicAddrSpace AddrSpace) const override; SIAtomicAddrSpace AddrSpace) const override;
bool enableNonTemporal(const MachineBasicBlock::iterator &MI) const override; bool enableNonTemporal(const MachineBasicBlock::iterator &MI) const override;
bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, bool insertCacheInvalidate(MachineBasicBlock::iterator &MI,
SIAtomicScope Scope, SIAtomicScope Scope,
SIAtomicAddrSpace AddrSpace, SIAtomicAddrSpace AddrSpace,
Position Pos) const override; Position Pos) const override;
bool insertWait(MachineBasicBlock::iterator &MI, bool insertWait(MachineBasicBlock::iterator &MI,
SIAtomicScope Scope, SIAtomicScope Scope,
SIAtomicAddrSpace AddrSpace, SIAtomicAddrSpace AddrSpace,
SIMemOp Op, SIMemOp Op,
bool IsCrossAddrSpaceOrdering, bool IsCrossAddrSpaceOrdering,
Position Pos) const override; Position Pos) const override;
}; };
class SIGfx7CacheControl : public SIGfx6CacheControl { class SIGfx7CacheControl : public SIGfx6CacheControl {
public: public:
SIGfx7CacheControl(const SISubtarget &ST) : SIGfx6CacheControl(ST) {}; SIGfx7CacheControl(const GCNSubtarget &ST) : SIGfx6CacheControl(ST) {};
bool insertCacheInvalidate(MachineBasicBlock::iterator &MI, bool insertCacheInvalidate(MachineBasicBlock::iterator &MI,
SIAtomicScope Scope, SIAtomicScope Scope,
SIAtomicAddrSpace AddrSpace, SIAtomicAddrSpace AddrSpace,
Position Pos) const override; Position Pos) const override;
}; };
▲ Show 20 Lines • Show All 243 Lines • ▼ Show 20 LinesOptional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
// Be conservative if there are no memory operands. // Be conservative if there are no memory operands.
if (MI->getNumMemOperands() == 0) if (MI->getNumMemOperands() == 0)
return SIMemOpInfo(); return SIMemOpInfo();
return constructFromMIWithMMO(MI); return constructFromMIWithMMO(MI);
} }
SICacheControl::SICacheControl(const SISubtarget &ST) { SICacheControl::SICacheControl(const GCNSubtarget &ST) {
TII = ST.getInstrInfo(); TII = ST.getInstrInfo();
IV = IsaInfo::getIsaVersion(ST.getFeatureBits()); IV = IsaInfo::getIsaVersion(ST.getFeatureBits());
} }
/* static */ /* static */
std::unique_ptr<SICacheControl> SICacheControl::create(const SISubtarget &ST) { std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST) {
AMDGPUSubtarget::Generation Generation = ST.getGeneration(); GCNSubtarget::Generation Generation = ST.getGeneration();
if (Generation <= AMDGPUSubtarget::SOUTHERN_ISLANDS) if (Generation <= GCNSubtarget::SOUTHERN_ISLANDS)
return make_unique<SIGfx6CacheControl>(ST); return make_unique<SIGfx6CacheControl>(ST);
return make_unique<SIGfx7CacheControl>(ST); return make_unique<SIGfx7CacheControl>(ST);
} }
bool SIGfx6CacheControl::enableLoadCacheBypass( bool SIGfx6CacheControl::enableLoadCacheBypass(
const MachineBasicBlock::iterator &MI, const MachineBasicBlock::iterator &MI,
SIAtomicScope Scope, SIAtomicScope Scope,
SIAtomicAddrSpace AddrSpace) const { SIAtomicAddrSpace AddrSpace) const {
▲ Show 20 Lines • Show All 381 Lines • ▼ Show 20 Linesbool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
return Changed; return Changed;
} }
bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) { bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
bool Changed = false; bool Changed = false;
SIMemOpAccess MOA(MF); SIMemOpAccess MOA(MF);
CC = SICacheControl::create(MF.getSubtarget<SISubtarget>()); CC = SICacheControl::create(MF.getSubtarget<GCNSubtarget>());
for (auto &MBB : MF) { for (auto &MBB : MF) {
for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) { for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic)) if (!(MI->getDesc().TSFlags & SIInstrFlags::maybeAtomic))
continue; continue;
if (const auto &MOI = MOA.getLoadInfo(MI)) if (const auto &MOI = MOA.getLoadInfo(MI))
Changed |= expandLoad(MOI.getValue(), MI); Changed |= expandLoad(MOI.getValue(), MI);
Show All 21 Lines

lib/Target/AMDGPU/SIOptimizeExecMasking.cpp

Show First 20 Lines • Show All 203 Lines • ▼ Show 20 Linesstatic bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
return false; return false;
} }
bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) { bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction())) if (skipFunction(MF.getFunction()))
return false; return false;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
// Optimize sequences emitted for control flow lowering. They are originally // Optimize sequences emitted for control flow lowering. They are originally
// emitted as the separate operations because spill code may need to be // emitted as the separate operations because spill code may need to be
// inserted for the saved copy of exec. // inserted for the saved copy of exec.
// //
// x = copy exec // x = copy exec
▲ Show 20 Lines • Show All 143 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp

Show First 20 Lines • Show All 101 Lines • ▼ Show 20 Lines if (!SaveExecInst || !isFullExecCopy(*SaveExecInst))
return nullptr; return nullptr;
return SaveExecInst; return SaveExecInst;
} }
bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) { bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction())) if (skipFunction(MF.getFunction()))
return false; return false;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI}); DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI});
bool Changed = false; bool Changed = false;
for (MachineBasicBlock &MBB : MF) { for (MachineBasicBlock &MBB : MF) {
▲ Show 20 Lines • Show All 136 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIPeepholeSDWA.cpp

Show First 20 Lines • Show All 84 Lines • ▼ Show 20 Linespublic:
SIPeepholeSDWA() : MachineFunctionPass(ID) { SIPeepholeSDWA() : MachineFunctionPass(ID) {
initializeSIPeepholeSDWAPass(*PassRegistry::getPassRegistry()); initializeSIPeepholeSDWAPass(*PassRegistry::getPassRegistry());
} }
bool runOnMachineFunction(MachineFunction &MF) override; bool runOnMachineFunction(MachineFunction &MF) override;
void matchSDWAOperands(MachineBasicBlock &MBB); void matchSDWAOperands(MachineBasicBlock &MBB);
std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI); std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
bool isConvertibleToSDWA(const MachineInstr &MI, const SISubtarget &ST) const; bool isConvertibleToSDWA(const MachineInstr &MI, const GCNSubtarget &ST) const;
bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands); bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
void legalizeScalarOperands(MachineInstr &MI, const SISubtarget &ST) const; void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
StringRef getPassName() const override { return "SI Peephole SDWA"; } StringRef getPassName() const override { return "SI Peephole SDWA"; }
void getAnalysisUsage(AnalysisUsage &AU) const override { void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG(); AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU);
} }
}; };
▲ Show 20 Lines • Show All 746 Lines • ▼ Show 20 Lines if (auto Operand = matchSDWAOperand(MI)) {
LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n'); LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n');
SDWAOperands[&MI] = std::move(Operand); SDWAOperands[&MI] = std::move(Operand);
++NumSDWAPatternsFound; ++NumSDWAPatternsFound;
} }
} }
} }
bool SIPeepholeSDWA::isConvertibleToSDWA(const MachineInstr &MI, bool SIPeepholeSDWA::isConvertibleToSDWA(const MachineInstr &MI,
const SISubtarget &ST) const { const GCNSubtarget &ST) const {
// Check if this is already an SDWA instruction // Check if this is already an SDWA instruction
unsigned Opc = MI.getOpcode(); unsigned Opc = MI.getOpcode();
if (TII->isSDWA(Opc)) if (TII->isSDWA(Opc))
return true; return true;
// Check if this instruction has opcode that supports SDWA // Check if this instruction has opcode that supports SDWA
if (AMDGPU::getSDWAOp(Opc) == -1) if (AMDGPU::getSDWAOp(Opc) == -1)
Opc = AMDGPU::getVOPe32(Opc); Opc = AMDGPU::getVOPe32(Opc);
▲ Show 20 Lines • Show All 210 Lines • ▼ Show 20 Linesbool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
MI.eraseFromParent(); MI.eraseFromParent();
return true; return true;
} }
// If an instruction was converted to SDWA it should not have immediates or SGPR // If an instruction was converted to SDWA it should not have immediates or SGPR
// operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs. // operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI, void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
const SISubtarget &ST) const { const GCNSubtarget &ST) const {
const MCInstrDesc &Desc = TII->get(MI.getOpcode()); const MCInstrDesc &Desc = TII->get(MI.getOpcode());
unsigned ConstantBusCount = 0; unsigned ConstantBusCount = 0;
for (MachineOperand &Op : MI.explicit_uses()) { for (MachineOperand &Op : MI.explicit_uses()) {
if (!Op.isImm() && !(Op.isReg() && !TRI->isVGPR(*MRI, Op.getReg()))) if (!Op.isImm() && !(Op.isReg() && !TRI->isVGPR(*MRI, Op.getReg())))
continue; continue;
unsigned I = MI.getOperandNo(&Op); unsigned I = MI.getOperandNo(&Op);
if (Desc.OpInfo[I].RegClass == -1 || if (Desc.OpInfo[I].RegClass == -1 ||
Show All 14 Lines for (MachineOperand &Op : MI.explicit_uses()) {
else if (Op.isReg()) else if (Op.isReg())
Copy.addReg(Op.getReg(), Op.isKill() ? RegState::Kill : 0, Copy.addReg(Op.getReg(), Op.isKill() ? RegState::Kill : 0,
Op.getSubReg()); Op.getSubReg());
Op.ChangeToRegister(VGPR, false); Op.ChangeToRegister(VGPR, false);
} }
} }
bool SIPeepholeSDWA::runOnMachineFunction(MachineFunction &MF) { bool SIPeepholeSDWA::runOnMachineFunction(MachineFunction &MF) {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (!ST.hasSDWA() || skipFunction(MF.getFunction())) if (!ST.hasSDWA() || skipFunction(MF.getFunction()))
return false; return false;
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo(); TRI = ST.getRegisterInfo();
TII = ST.getInstrInfo(); TII = ST.getInstrInfo();
Show All 34 Lines

lib/Target/AMDGPU/SIRegisterInfo.h

Show All 15 Lines
#define LLVM_LIB_TARGET_AMDGPU_SIREGISTERINFO_H #define LLVM_LIB_TARGET_AMDGPU_SIREGISTERINFO_H
#include "AMDGPURegisterInfo.h" #include "AMDGPURegisterInfo.h"
#include "SIDefines.h" #include "SIDefines.h"
#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h"
namespace llvm { namespace llvm {
class AMDGPUSubtarget; class GCNSubtarget;
class LiveIntervals; class LiveIntervals;
class MachineRegisterInfo; class MachineRegisterInfo;
class SISubtarget; class GCNSubtarget;
jveselyUnsubmitted
Done
ReplyInline Actions

This rename duplicates GCNSUbtarget declaration.

jvesely: This rename duplicates GCNSUbtarget declaration.
class SIMachineFunctionInfo; class SIMachineFunctionInfo;
class SIRegisterInfo final : public AMDGPURegisterInfo { class SIRegisterInfo final : public AMDGPURegisterInfo {
private: private:
unsigned SGPRSetID; unsigned SGPRSetID;
unsigned VGPRSetID; unsigned VGPRSetID;
BitVector SGPRPressureSets; BitVector SGPRPressureSets;
BitVector VGPRPressureSets; BitVector VGPRPressureSets;
bool SpillSGPRToVGPR; bool SpillSGPRToVGPR;
bool SpillSGPRToSMEM; bool SpillSGPRToSMEM;
void classifyPressureSet(unsigned PSetID, unsigned Reg, void classifyPressureSet(unsigned PSetID, unsigned Reg,
BitVector &PressureSets) const; BitVector &PressureSets) const;
public: public:
SIRegisterInfo(const SISubtarget &ST); SIRegisterInfo(const GCNSubtarget &ST);
bool spillSGPRToVGPR() const { bool spillSGPRToVGPR() const {
return SpillSGPRToVGPR; return SpillSGPRToVGPR;
} }
bool spillSGPRToSMEM() const { bool spillSGPRToSMEM() const {
return SpillSGPRToSMEM; return SpillSGPRToSMEM;
} }
▲ Show 20 Lines • Show All 197 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIRegisterInfo.cpp

Show First 20 Lines • Show All 50 Lines • ▼ Show 20 Linesstatic cl::opt<bool> EnableSpillSGPRToSMEM(
cl::init(false)); cl::init(false));
static cl::opt<bool> EnableSpillSGPRToVGPR( static cl::opt<bool> EnableSpillSGPRToVGPR(
"amdgpu-spill-sgpr-to-vgpr", "amdgpu-spill-sgpr-to-vgpr",
cl::desc("Enable spilling VGPRs to SGPRs"), cl::desc("Enable spilling VGPRs to SGPRs"),
cl::ReallyHidden, cl::ReallyHidden,
cl::init(true)); cl::init(true));
SIRegisterInfo::SIRegisterInfo(const SISubtarget &ST) : SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) :
AMDGPURegisterInfo(), AMDGPURegisterInfo(),
SGPRPressureSets(getNumRegPressureSets()), SGPRPressureSets(getNumRegPressureSets()),
VGPRPressureSets(getNumRegPressureSets()), VGPRPressureSets(getNumRegPressureSets()),
SpillSGPRToVGPR(false), SpillSGPRToVGPR(false),
SpillSGPRToSMEM(false) { SpillSGPRToSMEM(false) {
if (EnableSpillSGPRToSMEM && ST.hasScalarStores()) if (EnableSpillSGPRToSMEM && ST.hasScalarStores())
SpillSGPRToSMEM = true; SpillSGPRToSMEM = true;
else if (EnableSpillSGPRToVGPR) else if (EnableSpillSGPRToVGPR)
Show All 33 LinesSIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) :
assert(SGPRSetID < NumRegPressureSets && assert(SGPRSetID < NumRegPressureSets &&
VGPRSetID < NumRegPressureSets); VGPRSetID < NumRegPressureSets);
} }
unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg( unsigned SIRegisterInfo::reservedPrivateSegmentBufferReg(
const MachineFunction &MF) const { const MachineFunction &MF) const {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4; unsigned BaseIdx = alignDown(ST.getMaxNumSGPRs(MF), 4) - 4;
unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx)); unsigned BaseReg(AMDGPU::SGPR_32RegClass.getRegister(BaseIdx));
return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SReg_128RegClass); return getMatchingSuperReg(BaseReg, AMDGPU::sub0, &AMDGPU::SReg_128RegClass);
} }
static unsigned findPrivateSegmentWaveByteOffsetRegIndex(unsigned RegCount) { static unsigned findPrivateSegmentWaveByteOffsetRegIndex(unsigned RegCount) {
unsigned Reg; unsigned Reg;
// Try to place it in a hole after PrivateSegmentBufferReg. // Try to place it in a hole after PrivateSegmentBufferReg.
if (RegCount & 3) { if (RegCount & 3) {
// We cannot put the segment buffer in (Idx - 4) ... (Idx - 1) due to // We cannot put the segment buffer in (Idx - 4) ... (Idx - 1) due to
// alignment constraints, so we have a hole where can put the wave offset. // alignment constraints, so we have a hole where can put the wave offset.
Reg = RegCount - 1; Reg = RegCount - 1;
} else { } else {
// We can put the segment buffer in (Idx - 4) ... (Idx - 1) and put the // We can put the segment buffer in (Idx - 4) ... (Idx - 1) and put the
// wave offset before it. // wave offset before it.
Reg = RegCount - 5; Reg = RegCount - 5;
} }
return Reg; return Reg;
} }
unsigned SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg( unsigned SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg(
const MachineFunction &MF) const { const MachineFunction &MF) const {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
unsigned Reg = findPrivateSegmentWaveByteOffsetRegIndex(ST.getMaxNumSGPRs(MF)); unsigned Reg = findPrivateSegmentWaveByteOffsetRegIndex(ST.getMaxNumSGPRs(MF));
return AMDGPU::SGPR_32RegClass.getRegister(Reg); return AMDGPU::SGPR_32RegClass.getRegister(Reg);
} }
unsigned SIRegisterInfo::reservedStackPtrOffsetReg( unsigned SIRegisterInfo::reservedStackPtrOffsetReg(
const MachineFunction &MF) const { const MachineFunction &MF) const {
return AMDGPU::SGPR32; return AMDGPU::SGPR32;
} }
Show All 25 LinesBitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3); reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3);
reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5); reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5);
reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7); reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7);
reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9); reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9);
reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11); reserveRegisterTuples(Reserved, AMDGPU::TTMP10_TTMP11);
reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13); reserveRegisterTuples(Reserved, AMDGPU::TTMP12_TTMP13);
reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15); reserveRegisterTuples(Reserved, AMDGPU::TTMP14_TTMP15);
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF); unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); unsigned TotalNumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) { for (unsigned i = MaxNumSGPRs; i < TotalNumSGPRs; ++i) {
unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i); unsigned Reg = AMDGPU::SGPR_32RegClass.getRegister(i);
reserveRegisterTuples(Reserved, Reg); reserveRegisterTuples(Reserved, Reg);
} }
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines
} }
bool SIRegisterInfo::requiresFrameIndexReplacementScavenging( bool SIRegisterInfo::requiresFrameIndexReplacementScavenging(
const MachineFunction &MF) const { const MachineFunction &MF) const {
// m0 is needed for the scalar store offset. m0 is unallocatable, so we can't // m0 is needed for the scalar store offset. m0 is unallocatable, so we can't
// create a virtual register for it during frame index elimination, so the // create a virtual register for it during frame index elimination, so the
// scavenger is directly needed. // scavenger is directly needed.
return MF.getFrameInfo().hasStackObjects() && return MF.getFrameInfo().hasStackObjects() &&
MF.getSubtarget<SISubtarget>().hasScalarStores() && MF.getSubtarget<GCNSubtarget>().hasScalarStores() &&
MF.getInfo<SIMachineFunctionInfo>()->hasSpilledSGPRs(); MF.getInfo<SIMachineFunctionInfo>()->hasSpilledSGPRs();
} }
bool SIRegisterInfo::requiresVirtualBaseRegisters( bool SIRegisterInfo::requiresVirtualBaseRegisters(
const MachineFunction &) const { const MachineFunction &) const {
// There are no special dedicated stack or frame pointers. // There are no special dedicated stack or frame pointers.
return true; return true;
} }
Show All 38 Linesvoid SIRegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,
int64_t Offset) const { int64_t Offset) const {
MachineBasicBlock::iterator Ins = MBB->begin(); MachineBasicBlock::iterator Ins = MBB->begin();
DebugLoc DL; // Defaults to "unknown" DebugLoc DL; // Defaults to "unknown"
if (Ins != MBB->end()) if (Ins != MBB->end())
DL = Ins->getDebugLoc(); DL = Ins->getDebugLoc();
MachineFunction *MF = MBB->getParent(); MachineFunction *MF = MBB->getParent();
const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = Subtarget.getInstrInfo(); const SIInstrInfo *TII = Subtarget.getInstrInfo();
if (Offset == 0) { if (Offset == 0) {
BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg) BuildMI(*MBB, Ins, DL, TII->get(AMDGPU::V_MOV_B32_e32), BaseReg)
.addFrameIndex(FrameIdx); .addFrameIndex(FrameIdx);
return; return;
} }
Show All 12 Lines TII->getAddNoCarry(*MBB, Ins, DL, BaseReg)
.addReg(FIReg); .addReg(FIReg);
} }
void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg, void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg,
int64_t Offset) const { int64_t Offset) const {
MachineBasicBlock *MBB = MI.getParent(); MachineBasicBlock *MBB = MI.getParent();
MachineFunction *MF = MBB->getParent(); MachineFunction *MF = MBB->getParent();
const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &Subtarget = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = Subtarget.getInstrInfo(); const SIInstrInfo *TII = Subtarget.getInstrInfo();
#ifndef NDEBUG #ifndef NDEBUG
// FIXME: Is it possible to be storing a frame index to itself? // FIXME: Is it possible to be storing a frame index to itself?
bool SeenFI = false; bool SeenFI = false;
for (const MachineOperand &MO: MI.operands()) { for (const MachineOperand &MO: MI.operands()) {
if (MO.isFI()) { if (MO.isFI()) {
if (SeenFI) if (SeenFI)
▲ Show 20 Lines • Show All 170 Lines • ▼ Show 20 Linesvoid SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
bool IsKill, bool IsKill,
unsigned ScratchRsrcReg, unsigned ScratchRsrcReg,
unsigned ScratchOffsetReg, unsigned ScratchOffsetReg,
int64_t InstOffset, int64_t InstOffset,
MachineMemOperand *MMO, MachineMemOperand *MMO,
RegScavenger *RS) const { RegScavenger *RS) const {
MachineBasicBlock *MBB = MI->getParent(); MachineBasicBlock *MBB = MI->getParent();
MachineFunction *MF = MI->getParent()->getParent(); MachineFunction *MF = MI->getParent()->getParent();
const SISubtarget &ST = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const MachineFrameInfo &MFI = MF->getFrameInfo(); const MachineFrameInfo &MFI = MF->getFrameInfo();
const MCInstrDesc &Desc = TII->get(LoadStoreOp); const MCInstrDesc &Desc = TII->get(LoadStoreOp);
const DebugLoc &DL = MI->getDebugLoc(); const DebugLoc &DL = MI->getDebugLoc();
bool IsStore = Desc.mayStore(); bool IsStore = Desc.mayStore();
bool RanOutOfSGPRs = false; bool RanOutOfSGPRs = false;
▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Linesbool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
= MFI->getSGPRToVGPRSpills(Index); = MFI->getSGPRToVGPRSpills(Index);
bool SpillToVGPR = !VGPRSpills.empty(); bool SpillToVGPR = !VGPRSpills.empty();
if (OnlyToVGPR && !SpillToVGPR) if (OnlyToVGPR && !SpillToVGPR)
return false; return false;
MachineRegisterInfo &MRI = MF->getRegInfo(); MachineRegisterInfo &MRI = MF->getRegInfo();
const SISubtarget &ST = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
unsigned SuperReg = MI->getOperand(0).getReg(); unsigned SuperReg = MI->getOperand(0).getReg();
bool IsKill = MI->getOperand(0).isKill(); bool IsKill = MI->getOperand(0).isKill();
const DebugLoc &DL = MI->getDebugLoc(); const DebugLoc &DL = MI->getDebugLoc();
MachineFrameInfo &FrameInfo = MF->getFrameInfo(); MachineFrameInfo &FrameInfo = MF->getFrameInfo();
▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Linesbool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI,
ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills ArrayRef<SIMachineFunctionInfo::SpilledReg> VGPRSpills
= MFI->getSGPRToVGPRSpills(Index); = MFI->getSGPRToVGPRSpills(Index);
bool SpillToVGPR = !VGPRSpills.empty(); bool SpillToVGPR = !VGPRSpills.empty();
if (OnlyToVGPR && !SpillToVGPR) if (OnlyToVGPR && !SpillToVGPR)
return false; return false;
MachineFrameInfo &FrameInfo = MF->getFrameInfo(); MachineFrameInfo &FrameInfo = MF->getFrameInfo();
const SISubtarget &ST = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const DebugLoc &DL = MI->getDebugLoc(); const DebugLoc &DL = MI->getDebugLoc();
unsigned SuperReg = MI->getOperand(0).getReg(); unsigned SuperReg = MI->getOperand(0).getReg();
bool SpillToSMEM = spillSGPRToSMEM(); bool SpillToSMEM = spillSGPRToSMEM();
if (SpillToSMEM && OnlyToVGPR) if (SpillToSMEM && OnlyToVGPR)
return false; return false;
▲ Show 20 Lines • Show All 143 Lines • ▼ Show 20 Lines
void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
int SPAdj, unsigned FIOperandNum, int SPAdj, unsigned FIOperandNum,
RegScavenger *RS) const { RegScavenger *RS) const {
MachineFunction *MF = MI->getParent()->getParent(); MachineFunction *MF = MI->getParent()->getParent();
MachineRegisterInfo &MRI = MF->getRegInfo(); MachineRegisterInfo &MRI = MF->getRegInfo();
MachineBasicBlock *MBB = MI->getParent(); MachineBasicBlock *MBB = MI->getParent();
SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
MachineFrameInfo &FrameInfo = MF->getFrameInfo(); MachineFrameInfo &FrameInfo = MF->getFrameInfo();
const SISubtarget &ST = MF->getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
DebugLoc DL = MI->getDebugLoc(); DebugLoc DL = MI->getDebugLoc();
MachineOperand &FIOp = MI->getOperand(FIOperandNum); MachineOperand &FIOp = MI->getOperand(FIOperandNum);
int Index = MI->getOperand(FIOperandNum).getIndex(); int Index = MI->getOperand(FIOperandNum).getIndex();
switch (MI->getOpcode()) { switch (MI->getOpcode()) {
// SGPR register spill // SGPR register spill
▲ Show 20 Lines • Show All 525 Lines • ▼ Show 20 Lines if (SrcSize <= 32 || DstSize <= 32)
return true; return true;
return NewSize <= DstSize || NewSize <= SrcSize; return NewSize <= DstSize || NewSize <= SrcSize;
} }
unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
MachineFunction &MF) const { MachineFunction &MF) const {
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(), unsigned Occupancy = ST.getOccupancyWithLocalMemSize(MFI->getLDSSize(),
MF.getFunction()); MF.getFunction());
switch (RC->getID()) { switch (RC->getID()) {
default: default:
return AMDGPURegisterInfo::getRegPressureLimit(RC, MF); return AMDGPURegisterInfo::getRegPressureLimit(RC, MF);
case AMDGPU::VGPR_32RegClassID: case AMDGPU::VGPR_32RegClassID:
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIShrinkInstructions.cpp

Show First 20 Lines • Show All 286 Lines • ▼ Show 20 Linesstatic void shrinkScalarCompare(const SIInstrInfo *TII, MachineInstr &MI) {
} }
} }
bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) { bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction())) if (skipFunction(MF.getFunction()))
return false; return false;
MachineRegisterInfo &MRI = MF.getRegInfo(); MachineRegisterInfo &MRI = MF.getRegInfo();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo(); const SIInstrInfo *TII = ST.getInstrInfo();
const SIRegisterInfo &TRI = TII->getRegisterInfo(); const SIRegisterInfo &TRI = TII->getRegisterInfo();
std::vector<unsigned> I1Defs; std::vector<unsigned> I1Defs;
for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
BI != BE; ++BI) { BI != BE; ++BI) {
▲ Show 20 Lines • Show All 243 Lines • Show Last 20 Lines

lib/Target/AMDGPU/SIWholeQuadMode.cpp

Show First 20 Lines • Show All 843 Lines • ▼ Show 20 Lines
bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
Instructions.clear(); Instructions.clear();
Blocks.clear(); Blocks.clear();
LiveMaskQueries.clear(); LiveMaskQueries.clear();
LowerToCopyInstrs.clear(); LowerToCopyInstrs.clear();
CallingConv = MF.getFunction().getCallingConv(); CallingConv = MF.getFunction().getCallingConv();
const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo(); TII = ST.getInstrInfo();
TRI = &TII->getRegisterInfo(); TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo(); MRI = &MF.getRegInfo();
LIS = &getAnalysis<LiveIntervals>(); LIS = &getAnalysis<LiveIntervals>();
char GlobalFlags = analyzeFunction(MF); char GlobalFlags = analyzeFunction(MF);
unsigned LiveMaskReg = 0; unsigned LiveMaskReg = 0;
▲ Show 20 Lines • Show All 46 Lines • Show Last 20 Lines