This is an archive of the discontinued LLVM Phabricator instance.

Differential D73945

[NFC] Encapsulate MemOp logic
ClosedPublic

Authored by gchatelet on Feb 4 2020, 12:54 AM.

Details

Reviewers
courbet
Commits
rGb8144c053620: [NFC] Encapsulate MemOp logic
Summary

This patch simply introduces functions instead of directly accessing the fields.
This helps introducing additional check logic. A second patch will add simplifying functions.

Diff Detail

Event Timeline

gchatelet created this revision.Feb 4 2020, 12:54 AM
Herald added a project: Restricted Project. · View Herald TranscriptFeb 4 2020, 12:54 AM
Herald added subscribers: llvm-commits, kerbowa, jsji and 6 others. · View Herald Transcript
Harbormaster completed remote builds in B45665: Diff 242269.Feb 4 2020, 12:59 AM
Herald added a subscriber: • wuzish. · View Herald TranscriptFeb 4 2020, 12:59 AM
gchatelet updated this revision to Diff 242270.Feb 4 2020, 1:06 AM
  • Remove asserts for now
Harbormaster completed remote builds in B45666: Diff 242270.Feb 4 2020, 1:08 AM
courbet accepted this revision.Feb 4 2020, 1:12 AM
courbet added inline comments.
llvm/include/llvm/CodeGen/TargetLowering.h
115

This can no longer be zero, please update the comment.

124–125

ditto

This revision is now accepted and ready to land.Feb 4 2020, 1:12 AM
gchatelet updated this revision to Diff 242272.Feb 4 2020, 1:36 AM
  • Address comments
Harbormaster completed remote builds in B45667: Diff 242272.Feb 4 2020, 1:38 AM
Closed by commit rGb8144c053620: [NFC] Encapsulate MemOp logic (authored by gchatelet). · Explain WhyFeb 4 2020, 1:38 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
include/
llvm/
CodeGen/
64 lines
lib/
CodeGen/
GlobalISel/
12 lines
SelectionDAG/
12 lines
Target/
AArch64/
20 lines
AMDGPU/
5 lines
ARM/
10 lines
BPF/
2 lines
Hexagon/
12 lines
PowerPC/
16 lines
X86/
17 lines

Diff 242273

llvm/include/llvm/CodeGen/TargetLowering.h

Show First 20 Lines • Show All 102 Lines • ▼ Show 20 Lines enum Preference {
ILP, // Scheduling for ILP in low register pressure mode. ILP, // Scheduling for ILP in low register pressure mode.
VLIW // Scheduling for VLIW targets. VLIW // Scheduling for VLIW targets.
}; };
} // end namespace Sched } // end namespace Sched
// MemOp models a memory operation, either memset or memcpy/memmove. // MemOp models a memory operation, either memset or memcpy/memmove.
struct MemOp { struct MemOp {
private:
// Shared // Shared
uint64_t Size; uint64_t Size;
uint64_t DstAlign; // Specified alignment of the memory operation or zero if bool DstAlignCanChange; // true if destination alignment can satisfy any
// destination alignment can satisfy any constraint. // constraint.
courbetUnsubmitted
Not Done
ReplyInline Actions

This can no longer be zero, please update the comment.

courbet: This can no longer be zero, please update the comment.
Align DstAlign; // Specified alignment of the memory operation.
bool AllowOverlap; bool AllowOverlap;
// memset only // memset only
bool IsMemset; // If setthis memory operation is a memset. bool IsMemset; // If setthis memory operation is a memset.
bool ZeroMemset; // If set clears out memory with zeros. bool ZeroMemset; // If set clears out memory with zeros.
// memcpy only // memcpy only
bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
// constant so it does not need to be loaded. // constant so it does not need to be loaded.
uint64_t SrcAlign; // Inferred alignment of the source or zero if the memory Align SrcAlign; // Inferred alignment of the source or default value if the
courbetUnsubmitted
Not Done
ReplyInline Actions

ditto

courbet: ditto
// operation does not need to load the value. // memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
Align SrcAlign, bool IsVolatile, Align SrcAlign, bool IsVolatile,
bool MemcpyStrSrc = false) { bool MemcpyStrSrc = false) {
return { MemOp Op;
/*.Size =*/Size, Op.Size = Size;
/*.DstAlign =*/DstAlignCanChange ? 0 : DstAlign.value(), Op.DstAlignCanChange = DstAlignCanChange;
/*.AllowOverlap =*/!IsVolatile, Op.DstAlign = DstAlign;
/*.IsMemset =*/false, Op.AllowOverlap = !IsVolatile;
/*.ZeroMemset =*/false, Op.IsMemset = false;
/*.MemcpyStrSrc =*/MemcpyStrSrc, Op.ZeroMemset = false;
/*.SrcAlign =*/SrcAlign.value(), Op.MemcpyStrSrc = MemcpyStrSrc;
}; Op.SrcAlign = SrcAlign;
return Op;
} }
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
bool IsZeroMemset, bool IsVolatile) { bool IsZeroMemset, bool IsVolatile) {
return { MemOp Op;
/*.Size =*/Size, Op.Size = Size;
/*.DstAlign =*/DstAlignCanChange ? 0 : DstAlign.value(), Op.DstAlignCanChange = DstAlignCanChange;
/*.AllowOverlap =*/!IsVolatile, Op.DstAlign = DstAlign;
/*.IsMemset =*/true, Op.AllowOverlap = !IsVolatile;
/*.ZeroMemset =*/IsZeroMemset, Op.IsMemset = true;
/*.MemcpyStrSrc =*/false, Op.ZeroMemset = IsZeroMemset;
/*.SrcAlign =*/0, Op.MemcpyStrSrc = false;
}; return Op;
} }
uint64_t size() const { return Size; }
uint64_t getDstAlign() const {
return DstAlignCanChange ? 0 : DstAlign.value();
}
bool allowOverlap() const { return AllowOverlap; }
bool isMemset() const { return IsMemset; }
bool isMemcpy() const { return !IsMemset; }
bool isZeroMemset() const { return ZeroMemset; }
bool isMemcpyStrSrc() const { return MemcpyStrSrc; }
uint64_t getSrcAlign() const { return isMemset() ? 0 : SrcAlign.value(); }
}; };
/// This base class for TargetLowering contains the SelectionDAG-independent /// This base class for TargetLowering contains the SelectionDAG-independent
/// parts that can be used from the rest of CodeGen. /// parts that can be used from the rest of CodeGen.
class TargetLoweringBase { class TargetLoweringBase {
public: public:
/// This enum indicates whether operations are valid for a target, and if not, /// This enum indicates whether operations are valid for a target, and if not,
/// what action should be used to make them valid. /// what action should be used to make them valid.
▲ Show 20 Lines • Show All 4,190 Lines • Show Last 20 Lines

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

Show First 20 Lines • Show All 854 Lines • ▼ Show 20 Lines
// Returns a list of types to use for memory op lowering in MemOps. A partial // Returns a list of types to use for memory op lowering in MemOps. A partial
// port of findOptimalMemOpLowering in TargetLowering. // port of findOptimalMemOpLowering in TargetLowering.
static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps, static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps,
unsigned Limit, const MemOp &Op, unsigned Limit, const MemOp &Op,
unsigned DstAS, unsigned SrcAS, unsigned DstAS, unsigned SrcAS,
const AttributeList &FuncAttributes, const AttributeList &FuncAttributes,
const TargetLowering &TLI) { const TargetLowering &TLI) {
if (Op.SrcAlign != 0 && Op.SrcAlign < Op.DstAlign) if (Op.getSrcAlign() != 0 && Op.getSrcAlign() < Op.getDstAlign())
return false; return false;
LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes); LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes);
if (Ty == LLT()) { if (Ty == LLT()) {
// Use the largest scalar type whose alignment constraints are satisfied. // Use the largest scalar type whose alignment constraints are satisfied.
// We only need to check DstAlign here as SrcAlign is always greater or // We only need to check DstAlign here as SrcAlign is always greater or
// equal to DstAlign (or zero). // equal to DstAlign (or zero).
Ty = LLT::scalar(64); Ty = LLT::scalar(64);
while (Op.DstAlign && Op.DstAlign < Ty.getSizeInBytes() && while (Op.getDstAlign() && Op.getDstAlign() < Ty.getSizeInBytes() &&
!TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.DstAlign)) !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign()))
Ty = LLT::scalar(Ty.getSizeInBytes()); Ty = LLT::scalar(Ty.getSizeInBytes());
assert(Ty.getSizeInBits() > 0 && "Could not find valid type"); assert(Ty.getSizeInBits() > 0 && "Could not find valid type");
// FIXME: check for the largest legal type we can load/store to. // FIXME: check for the largest legal type we can load/store to.
} }
unsigned NumMemOps = 0; unsigned NumMemOps = 0;
auto Size = Op.Size; auto Size = Op.size();
while (Size != 0) { while (Size != 0) {
unsigned TySize = Ty.getSizeInBytes(); unsigned TySize = Ty.getSizeInBytes();
while (TySize > Size) { while (TySize > Size) {
// For now, only use non-vector load / store's for the left-over pieces. // For now, only use non-vector load / store's for the left-over pieces.
LLT NewTy = Ty; LLT NewTy = Ty;
// FIXME: check for mem op safety and legality of the types. Not all of // FIXME: check for mem op safety and legality of the types. Not all of
// SDAGisms map cleanly to GISel concepts. // SDAGisms map cleanly to GISel concepts.
if (NewTy.isVector()) if (NewTy.isVector())
NewTy = NewTy.getSizeInBits() > 64 ? LLT::scalar(64) : LLT::scalar(32); NewTy = NewTy.getSizeInBits() > 64 ? LLT::scalar(64) : LLT::scalar(32);
NewTy = LLT::scalar(PowerOf2Floor(NewTy.getSizeInBits() - 1)); NewTy = LLT::scalar(PowerOf2Floor(NewTy.getSizeInBits() - 1));
unsigned NewTySize = NewTy.getSizeInBytes(); unsigned NewTySize = NewTy.getSizeInBytes();
assert(NewTySize > 0 && "Could not find appropriate type"); assert(NewTySize > 0 && "Could not find appropriate type");
// If the new LLT cannot cover all of the remaining bits, then consider // If the new LLT cannot cover all of the remaining bits, then consider
// issuing a (or a pair of) unaligned and overlapping load / store. // issuing a (or a pair of) unaligned and overlapping load / store.
bool Fast; bool Fast;
// Need to get a VT equivalent for allowMisalignedMemoryAccesses(). // Need to get a VT equivalent for allowMisalignedMemoryAccesses().
MVT VT = getMVTForLLT(Ty); MVT VT = getMVTForLLT(Ty);
if (NumMemOps && Op.AllowOverlap && NewTySize < Size && if (NumMemOps && Op.allowOverlap() && NewTySize < Size &&
TLI.allowsMisalignedMemoryAccesses( TLI.allowsMisalignedMemoryAccesses(
VT, DstAS, Op.DstAlign, MachineMemOperand::MONone, &Fast) && VT, DstAS, Op.getDstAlign(), MachineMemOperand::MONone, &Fast) &&
Fast) Fast)
TySize = Size; TySize = Size;
else { else {
Ty = NewTy; Ty = NewTy;
TySize = NewTySize; TySize = NewTySize;
} }
} }
▲ Show 20 Lines • Show All 500 Lines • Show Last 20 Lines

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 180 Lines • ▼ Show 20 Linesbool TargetLowering::findOptimalMemOpLowering(
unsigned SrcAS, const AttributeList &FuncAttributes) const { unsigned SrcAS, const AttributeList &FuncAttributes) const {
// If 'SrcAlign' is zero, that means the memory operation does not need to // If 'SrcAlign' is zero, that means the memory operation does not need to
// load the value, i.e. memset or memcpy from constant string. Otherwise, // load the value, i.e. memset or memcpy from constant string. Otherwise,
// it's the inferred alignment of the source. 'DstAlign', on the other hand, // it's the inferred alignment of the source. 'DstAlign', on the other hand,
// is the specified alignment of the memory operation. If it is zero, that // is the specified alignment of the memory operation. If it is zero, that
// means it's possible to change the alignment of the destination. // means it's possible to change the alignment of the destination.
// 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
// not need to be loaded. // not need to be loaded.
if (!(Op.SrcAlign == 0 || Op.SrcAlign >= Op.DstAlign)) if (!(Op.getSrcAlign() == 0 || Op.getSrcAlign() >= Op.getDstAlign()))
return false; return false;
EVT VT = getOptimalMemOpType(Op, FuncAttributes); EVT VT = getOptimalMemOpType(Op, FuncAttributes);
if (VT == MVT::Other) { if (VT == MVT::Other) {
// Use the largest integer type whose alignment constraints are satisfied. // Use the largest integer type whose alignment constraints are satisfied.
// We only need to check DstAlign here as SrcAlign is always greater or // We only need to check DstAlign here as SrcAlign is always greater or
// equal to DstAlign (or zero). // equal to DstAlign (or zero).
VT = MVT::i64; VT = MVT::i64;
while (Op.DstAlign && Op.DstAlign < VT.getSizeInBits() / 8 && while (Op.getDstAlign() && Op.getDstAlign() < VT.getSizeInBits() / 8 &&
!allowsMisalignedMemoryAccesses(VT, DstAS, Op.DstAlign)) !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
assert(VT.isInteger()); assert(VT.isInteger());
// Find the largest legal integer type. // Find the largest legal integer type.
MVT LVT = MVT::i64; MVT LVT = MVT::i64;
while (!isTypeLegal(LVT)) while (!isTypeLegal(LVT))
LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
assert(LVT.isInteger()); assert(LVT.isInteger());
// If the type we've chosen is larger than the largest legal integer type // If the type we've chosen is larger than the largest legal integer type
// then use that instead. // then use that instead.
if (VT.bitsGT(LVT)) if (VT.bitsGT(LVT))
VT = LVT; VT = LVT;
} }
unsigned NumMemOps = 0; unsigned NumMemOps = 0;
auto Size = Op.Size; auto Size = Op.size();
while (Size != 0) { while (Size != 0) {
unsigned VTSize = VT.getSizeInBits() / 8; unsigned VTSize = VT.getSizeInBits() / 8;
while (VTSize > Size) { while (VTSize > Size) {
// For now, only use non-vector load / store's for the left-over pieces. // For now, only use non-vector load / store's for the left-over pieces.
EVT NewVT = VT; EVT NewVT = VT;
unsigned NewVTSize; unsigned NewVTSize;
bool Found = false; bool Found = false;
Show All 18 Lines while (VTSize > Size) {
break; break;
} while (!isSafeMemOpType(NewVT.getSimpleVT())); } while (!isSafeMemOpType(NewVT.getSimpleVT()));
} }
NewVTSize = NewVT.getSizeInBits() / 8; NewVTSize = NewVT.getSizeInBits() / 8;
// If the new VT cannot cover all of the remaining bits, then consider // If the new VT cannot cover all of the remaining bits, then consider
// issuing a (or a pair of) unaligned and overlapping load / store. // issuing a (or a pair of) unaligned and overlapping load / store.
bool Fast; bool Fast;
if (NumMemOps && Op.AllowOverlap && NewVTSize < Size && if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
allowsMisalignedMemoryAccesses(VT, DstAS, Op.DstAlign, allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign(),
MachineMemOperand::MONone, &Fast) && MachineMemOperand::MONone, &Fast) &&
Fast) Fast)
VTSize = Size; VTSize = Size;
else { else {
VT = NewVT; VT = NewVT;
VTSize = NewVTSize; VTSize = NewVTSize;
} }
} }
▲ Show 20 Lines • Show All 7,405 Lines • Show Last 20 Lines

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 9,430 Lines • ▼ Show 20 LinesEVT AArch64TargetLowering::getOptimalMemOpType(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
bool CanImplicitFloat = bool CanImplicitFloat =
!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat); !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat);
bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat; bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat; bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
// Only use AdvSIMD to implement memset of 32-byte and above. It would have // Only use AdvSIMD to implement memset of 32-byte and above. It would have
// taken one instruction to materialize the v2i64 zero and one store (with // taken one instruction to materialize the v2i64 zero and one store (with
// restrictive addressing mode). Just do i64 stores. // restrictive addressing mode). Just do i64 stores.
bool IsSmallMemset = Op.IsMemset && Op.Size < 32; bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
auto AlignmentIsAcceptable = [&](EVT VT, unsigned AlignCheck) { auto AlignmentIsAcceptable = [&](EVT VT, unsigned AlignCheck) {
if (memOpAlign(Op.SrcAlign, Op.DstAlign, AlignCheck)) if (memOpAlign(Op.getSrcAlign(), Op.getDstAlign(), AlignCheck))
return true; return true;
bool Fast; bool Fast;
return allowsMisalignedMemoryAccesses(VT, 0, 1, MachineMemOperand::MONone, return allowsMisalignedMemoryAccesses(VT, 0, 1, MachineMemOperand::MONone,
&Fast) && &Fast) &&
Fast; Fast;
}; };
if (CanUseNEON && Op.IsMemset && !IsSmallMemset && if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
AlignmentIsAcceptable(MVT::v2i64, 16)) AlignmentIsAcceptable(MVT::v2i64, 16))
return MVT::v2i64; return MVT::v2i64;
if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, 16)) if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, 16))
return MVT::f128; return MVT::f128;
if (Op.Size >= 8 && AlignmentIsAcceptable(MVT::i64, 8)) if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, 8))
return MVT::i64; return MVT::i64;
if (Op.Size >= 4 && AlignmentIsAcceptable(MVT::i32, 4)) if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, 4))
return MVT::i32; return MVT::i32;
return MVT::Other; return MVT::Other;
} }
LLT AArch64TargetLowering::getOptimalMemOpLLT( LLT AArch64TargetLowering::getOptimalMemOpLLT(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
bool CanImplicitFloat = bool CanImplicitFloat =
!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat); !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat);
bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat; bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat; bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
// Only use AdvSIMD to implement memset of 32-byte and above. It would have // Only use AdvSIMD to implement memset of 32-byte and above. It would have
// taken one instruction to materialize the v2i64 zero and one store (with // taken one instruction to materialize the v2i64 zero and one store (with
// restrictive addressing mode). Just do i64 stores. // restrictive addressing mode). Just do i64 stores.
bool IsSmallMemset = Op.IsMemset && Op.Size < 32; bool IsSmallMemset = Op.isMemset() && Op.size() < 32;
auto AlignmentIsAcceptable = [&](EVT VT, unsigned AlignCheck) { auto AlignmentIsAcceptable = [&](EVT VT, unsigned AlignCheck) {
if (memOpAlign(Op.SrcAlign, Op.DstAlign, AlignCheck)) if (memOpAlign(Op.getSrcAlign(), Op.getDstAlign(), AlignCheck))
return true; return true;
bool Fast; bool Fast;
return allowsMisalignedMemoryAccesses(VT, 0, 1, MachineMemOperand::MONone, return allowsMisalignedMemoryAccesses(VT, 0, 1, MachineMemOperand::MONone,
&Fast) && &Fast) &&
Fast; Fast;
}; };
if (CanUseNEON && Op.IsMemset && !IsSmallMemset && if (CanUseNEON && Op.isMemset() && !IsSmallMemset &&
AlignmentIsAcceptable(MVT::v2i64, 16)) AlignmentIsAcceptable(MVT::v2i64, 16))
return LLT::vector(2, 64); return LLT::vector(2, 64);
if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, 16)) if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, 16))
return LLT::scalar(128); return LLT::scalar(128);
if (Op.Size >= 8 && AlignmentIsAcceptable(MVT::i64, 8)) if (Op.size() >= 8 && AlignmentIsAcceptable(MVT::i64, 8))
return LLT::scalar(64); return LLT::scalar(64);
if (Op.Size >= 4 && AlignmentIsAcceptable(MVT::i32, 4)) if (Op.size() >= 4 && AlignmentIsAcceptable(MVT::i32, 4))
return LLT::scalar(32); return LLT::scalar(32);
return LLT(); return LLT();
} }
// 12-bit optionally shifted immediates are legal for adds. // 12-bit optionally shifted immediates are legal for adds.
bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const { bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
if (Immed == std::numeric_limits<int64_t>::min()) { if (Immed == std::numeric_limits<int64_t>::min()) {
LLVM_DEBUG(dbgs() << "Illegal add imm " << Immed LLVM_DEBUG(dbgs() << "Illegal add imm " << Immed
▲ Show 20 Lines • Show All 4,048 Lines • Show Last 20 Lines

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 1,320 Lines • ▼ Show 20 Lines
EVT SITargetLowering::getOptimalMemOpType( EVT SITargetLowering::getOptimalMemOpType(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
// FIXME: Should account for address space here. // FIXME: Should account for address space here.
// The default fallback uses the private pointer size as a guess for a type to // The default fallback uses the private pointer size as a guess for a type to
// use. Make sure we switch these to 64-bit accesses. // use. Make sure we switch these to 64-bit accesses.
if (Op.Size >= 16 && Op.DstAlign >= 4) // XXX: Should only do for global if (Op.size() >= 16 &&
Op.getDstAlign() >= 4) // XXX: Should only do for global
return MVT::v4i32; return MVT::v4i32;
if (Op.Size >= 8 && Op.DstAlign >= 4) if (Op.size() >= 8 && Op.getDstAlign() >= 4)
return MVT::v2i32; return MVT::v2i32;
// Use the default. // Use the default.
return MVT::Other; return MVT::Other;
} }
bool SITargetLowering::isNoopAddrSpaceCast(unsigned SrcAS, bool SITargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
unsigned DestAS) const { unsigned DestAS) const {
▲ Show 20 Lines • Show All 9,646 Lines • Show Last 20 Lines

llvm/lib/Target/ARM/ARMISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 15,021 Lines • ▼ Show 20 Linesstatic bool memOpAlign(unsigned DstAlign, unsigned SrcAlign,
unsigned AlignCheck) { unsigned AlignCheck) {
return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) && return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) &&
(DstAlign == 0 || DstAlign % AlignCheck == 0)); (DstAlign == 0 || DstAlign % AlignCheck == 0));
} }
EVT ARMTargetLowering::getOptimalMemOpType( EVT ARMTargetLowering::getOptimalMemOpType(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
// See if we can use NEON instructions for this... // See if we can use NEON instructions for this...
if ((!Op.IsMemset || Op.ZeroMemset) && Subtarget->hasNEON() && if ((!Op.isMemset() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
bool Fast; bool Fast;
if (Op.Size >= 16 && if (Op.size() >= 16 &&
(memOpAlign(Op.SrcAlign, Op.DstAlign, 16) || (memOpAlign(Op.getSrcAlign(), Op.getDstAlign(), 16) ||
(allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1, (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1,
MachineMemOperand::MONone, &Fast) && MachineMemOperand::MONone, &Fast) &&
Fast))) { Fast))) {
return MVT::v2f64; return MVT::v2f64;
} else if (Op.Size >= 8 && } else if (Op.size() >= 8 &&
(memOpAlign(Op.SrcAlign, Op.DstAlign, 8) || (memOpAlign(Op.getSrcAlign(), Op.getDstAlign(), 8) ||
(allowsMisalignedMemoryAccesses( (allowsMisalignedMemoryAccesses(
MVT::f64, 0, 1, MachineMemOperand::MONone, &Fast) && MVT::f64, 0, 1, MachineMemOperand::MONone, &Fast) &&
Fast))) { Fast))) {
return MVT::f64; return MVT::f64;
} }
} }
// Let the target-independent logic figure it out. // Let the target-independent logic figure it out.
▲ Show 20 Lines • Show All 2,635 Lines • Show Last 20 Lines

llvm/lib/Target/BPF/BPFISelLowering.h

Show First 20 Lines • Show All 95 Lines • ▼ Show 20 Linesprivate:
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs, const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
SelectionDAG &DAG) const override; SelectionDAG &DAG) const override;
EVT getOptimalMemOpType(const MemOp &Op, EVT getOptimalMemOpType(const MemOp &Op,
const AttributeList &FuncAttributes) const override { const AttributeList &FuncAttributes) const override {
return Op.Size >= 8 ? MVT::i64 : MVT::i32; return Op.size() >= 8 ? MVT::i64 : MVT::i32;
} }
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
Type *Ty) const override { Type *Ty) const override {
return true; return true;
} }
unsigned EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB, unsigned Reg, unsigned EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB, unsigned Reg,
Show All 10 Lines

llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

Show First 20 Lines • Show All 3,379 Lines • ▼ Show 20 Lines
/// determined using generic target-independent logic. /// determined using generic target-independent logic.
EVT HexagonTargetLowering::getOptimalMemOpType( EVT HexagonTargetLowering::getOptimalMemOpType(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
auto Aligned = [](unsigned GivenA, unsigned MinA) -> bool { auto Aligned = [](unsigned GivenA, unsigned MinA) -> bool {
return (GivenA % MinA) == 0; return (GivenA % MinA) == 0;
}; };
if (Op.Size >= 8 && Aligned(Op.DstAlign, 8) && if (Op.size() >= 8 && Aligned(Op.getDstAlign(), 8) &&
(Op.IsMemset || Aligned(Op.SrcAlign, 8))) (Op.isMemset() || Aligned(Op.getSrcAlign(), 8)))
return MVT::i64; return MVT::i64;
if (Op.Size >= 4 && Aligned(Op.DstAlign, 4) && if (Op.size() >= 4 && Aligned(Op.getDstAlign(), 4) &&
(Op.IsMemset || Aligned(Op.SrcAlign, 4))) (Op.isMemset() || Aligned(Op.getSrcAlign(), 4)))
return MVT::i32; return MVT::i32;
if (Op.Size >= 2 && Aligned(Op.DstAlign, 2) && if (Op.size() >= 2 && Aligned(Op.getDstAlign(), 2) &&
(Op.IsMemset || Aligned(Op.SrcAlign, 2))) (Op.isMemset() || Aligned(Op.getSrcAlign(), 2)))
return MVT::i16; return MVT::i16;
return MVT::Other; return MVT::Other;
} }
bool HexagonTargetLowering::allowsMisalignedMemoryAccesses( bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned AS, unsigned Align, MachineMemOperand::Flags Flags, EVT VT, unsigned AS, unsigned Align, MachineMemOperand::Flags Flags,
bool *Fast) const { bool *Fast) const {
▲ Show 20 Lines • Show All 110 Lines • Show Last 20 Lines

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 15,072 Lines • ▼ Show 20 Lines
/// It returns EVT::Other if the type should be determined using generic /// It returns EVT::Other if the type should be determined using generic
/// target-independent logic. /// target-independent logic.
EVT PPCTargetLowering::getOptimalMemOpType( EVT PPCTargetLowering::getOptimalMemOpType(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { if (getTargetMachine().getOptLevel() != CodeGenOpt::None) {
// When expanding a memset, require at least two QPX instructions to cover // When expanding a memset, require at least two QPX instructions to cover
// the cost of loading the value to be stored from the constant pool. // the cost of loading the value to be stored from the constant pool.
if (Subtarget.hasQPX() && Op.Size >= 32 && if (Subtarget.hasQPX() && Op.size() >= 32 &&
(!Op.IsMemset || Op.Size >= 64) && (!Op.isMemset() || Op.size() >= 64) &&
(!Op.SrcAlign || Op.SrcAlign >= 32) && (!Op.getSrcAlign() || Op.getSrcAlign() >= 32) &&
(!Op.DstAlign || Op.DstAlign >= 32) && (!Op.getDstAlign() || Op.getDstAlign() >= 32) &&
!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
return MVT::v4f64; return MVT::v4f64;
} }
// We should use Altivec/VSX loads and stores when available. For unaligned // We should use Altivec/VSX loads and stores when available. For unaligned
// addresses, unaligned VSX loads are only fast starting with the P8. // addresses, unaligned VSX loads are only fast starting with the P8.
if (Subtarget.hasAltivec() && Op.Size >= 16 && if (Subtarget.hasAltivec() && Op.size() >= 16 &&
(((!Op.SrcAlign || Op.SrcAlign >= 16) && (((!Op.getSrcAlign() || Op.getSrcAlign() >= 16) &&
(!Op.DstAlign || Op.DstAlign >= 16)) || (!Op.getDstAlign() || Op.getDstAlign() >= 16)) ||
((Op.IsMemset && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) ((Op.isMemset() && Subtarget.hasVSX()) || Subtarget.hasP8Vector())))
return MVT::v4i32; return MVT::v4i32;
} }
if (Subtarget.isPPC64()) { if (Subtarget.isPPC64()) {
return MVT::i64; return MVT::i64;
} }
return MVT::i32; return MVT::i32;
▲ Show 20 Lines • Show All 725 Lines • Show Last 20 Lines

llvm/lib/Target/X86/X86ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 2,246 Lines • ▼ Show 20 Lines
/// It returns EVT::Other if the type should be determined using generic /// It returns EVT::Other if the type should be determined using generic
/// target-independent logic. /// target-independent logic.
/// For vector ops we check that the overall size isn't larger than our /// For vector ops we check that the overall size isn't larger than our
/// preferred vector width. /// preferred vector width.
EVT X86TargetLowering::getOptimalMemOpType( EVT X86TargetLowering::getOptimalMemOpType(
const MemOp &Op, const AttributeList &FuncAttributes) const { const MemOp &Op, const AttributeList &FuncAttributes) const {
if (!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { if (!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
if (Op.Size >= 16 && (!Subtarget.isUnalignedMem16Slow() || if (Op.size() >= 16 &&
((Op.DstAlign == 0 || Op.DstAlign >= 16) && (!Subtarget.isUnalignedMem16Slow() ||
(Op.SrcAlign == 0 || Op.SrcAlign >= 16)))) { ((Op.getDstAlign() == 0 || Op.getDstAlign() >= 16) &&
(Op.getSrcAlign() == 0 || Op.getSrcAlign() >= 16)))) {
// FIXME: Check if unaligned 64-byte accesses are slow. // FIXME: Check if unaligned 64-byte accesses are slow.
if (Op.Size >= 64 && Subtarget.hasAVX512() && if (Op.size() >= 64 && Subtarget.hasAVX512() &&
(Subtarget.getPreferVectorWidth() >= 512)) { (Subtarget.getPreferVectorWidth() >= 512)) {
return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32; return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
} }
// FIXME: Check if unaligned 32-byte accesses are slow. // FIXME: Check if unaligned 32-byte accesses are slow.
if (Op.Size >= 32 && Subtarget.hasAVX() && if (Op.size() >= 32 && Subtarget.hasAVX() &&
(Subtarget.getPreferVectorWidth() >= 256)) { (Subtarget.getPreferVectorWidth() >= 256)) {
// Although this isn't a well-supported type for AVX1, we'll let // Although this isn't a well-supported type for AVX1, we'll let
// legalization and shuffle lowering produce the optimal codegen. If we // legalization and shuffle lowering produce the optimal codegen. If we
// choose an optimal type with a vector element larger than a byte, // choose an optimal type with a vector element larger than a byte,
// getMemsetStores() may create an intermediate splat (using an integer // getMemsetStores() may create an intermediate splat (using an integer
// multiply) before we splat as a vector. // multiply) before we splat as a vector.
return MVT::v32i8; return MVT::v32i8;
} }
if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128)) if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128))
return MVT::v16i8; return MVT::v16i8;
// TODO: Can SSE1 handle a byte vector? // TODO: Can SSE1 handle a byte vector?
// If we have SSE1 registers we should be able to use them. // If we have SSE1 registers we should be able to use them.
if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) && if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
(Subtarget.getPreferVectorWidth() >= 128)) (Subtarget.getPreferVectorWidth() >= 128))
return MVT::v4f32; return MVT::v4f32;
} else if ((!Op.IsMemset || Op.ZeroMemset) && !Op.MemcpyStrSrc && } else if ((!Op.isMemset() || Op.isZeroMemset()) && !Op.isMemcpyStrSrc() &&
Op.Size >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) { Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
// Do not use f64 to lower memcpy if source is string constant. It's // Do not use f64 to lower memcpy if source is string constant. It's
// better to use i32 to avoid the loads. // better to use i32 to avoid the loads.
// Also, do not use f64 to lower memset unless this is a memset of zeros. // Also, do not use f64 to lower memset unless this is a memset of zeros.
// The gymnastics of splatting a byte value into an XMM register and then // The gymnastics of splatting a byte value into an XMM register and then
// only using 8-byte stores (because this is a CPU with slow unaligned // only using 8-byte stores (because this is a CPU with slow unaligned
// 16-byte accesses) makes that a loser. // 16-byte accesses) makes that a loser.
return MVT::f64; return MVT::f64;
} }
} }
// This is a compromise. If we reach here, unaligned accesses may be slow on // This is a compromise. If we reach here, unaligned accesses may be slow on
// this target. However, creating smaller, aligned accesses could be even // this target. However, creating smaller, aligned accesses could be even
// slower and would certainly be a lot more code. // slower and would certainly be a lot more code.
if (Subtarget.is64Bit() && Op.Size >= 8) if (Subtarget.is64Bit() && Op.size() >= 8)
return MVT::i64; return MVT::i64;
return MVT::i32; return MVT::i32;
} }
bool X86TargetLowering::isSafeMemOpType(MVT VT) const { bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
if (VT == MVT::f32) if (VT == MVT::f32)
return X86ScalarSSEf32; return X86ScalarSSEf32;
else if (VT == MVT::f64) else if (VT == MVT::f64)
▲ Show 20 Lines • Show All 45,254 Lines • Show Last 20 Lines