Remove GlobalValue::getAlignment().

[efriedma]

Summary

This function is deceptive at best; it's basically impossible to handle the return value correctly. If you have an arbitrary GlobalValue and you want to determine the alignment of that pointer, Value::getPointerAlignment() returns the correct value. If you want the actual declared alignment of a function or variable, GlobalObject::getAlignment() returns that.

This patch switches all the users of GlobalValue::getAlignment() to an appropriate alternative.

There are target-specific changes to AArch64, AMDGPU, PowerPC, SystemZ, and XCore; the changes seem reasonable to me, but probably could use a second look.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

1319–4587

These look fine

[nigelp-xmos]

llvm/lib/Target/XCore/XCoreISelLowering.cpp

438

Looks good on XCore. (I am not an approver. I have only reviewed for XCore.)

[jdoerfert]

Minor note. I think if we don't get a complaint this is OK.

llvm/lib/IR/Globals.cpp

113

Yeah, this seems wrong.

llvm/lib/LTO/LTOModule.cpp

418

Isn't 1 the right choice here?

[efriedma]

llvm/lib/LTO/LTOModule.cpp

418

I don't think the difference between 1 and 0 has any practical effect, the way the code is currently written. I guess I can clean this up to use Align, though.

[efriedma]

Address review comment

[jdoerfert]

LGTM.

llvm/include/llvm/CodeGen/AsmPrinter.h

public:
   void emitVisibility(MCSymbol *Sym, unsigned Visibility,
                       bool IsDefinition = true) const;
 
   /// This emits linkage information about \p GVSym based on \p GV, if this is
   /// supported by the target.
   void emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
 
   /// Return the alignment for the specified \p GV.
-  static Align getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
+  static Align getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
                               Align InAlign = Align(1));
 
 private:
   /// Private state for PrintSpecial()
   // Assign a unique ID to this machine instruction.
   mutable const MachineInstr *LastMI = nullptr;
   mutable unsigned LastFn = 0;
   mutable unsigned Counter = ~0U;

llvm/include/llvm/IR/GlobalObject.h

public:
   GlobalObject(const GlobalObject &) = delete;
 
   /// FIXME: Remove this function once transition to Align is over.
   unsigned getAlignment() const {
     MaybeAlign Align = getAlign();
     return Align ? Align->value() : 0;
   }
 
+  /// Returns the alignment of the given variable or function.
+  ///
+  /// Note that for functions this is the alignment of the code, not the
+  /// alignment of a function pointer.
   MaybeAlign getAlign() const {
     unsigned Data = getGlobalValueSubClassData();
     unsigned AlignmentData = Data & AlignmentMask;
     return decodeMaybeAlign(AlignmentData);
   }
 
   void setAlignment(MaybeAlign Align);
 
public:
 
   /// Copy metadata from Src, adjusting offsets by Offset.
   void copyMetadata(const GlobalObject *Src, unsigned Offset);
 
   void addTypeMetadata(unsigned Offset, Metadata *TypeID);
   void setVCallVisibilityMetadata(VCallVisibility Visibility);
   VCallVisibility getVCallVisibility() const;
 
+  /// Returns true if the alignment of the value can be unilaterally
+  /// increased.
+  ///
+  /// Note that for functions this is the alignment of the code, not the
+  /// alignment of a function pointer.
+  bool canIncreaseAlignment() const;
+
 protected:
   void copyAttributesFrom(const GlobalObject *Src);
 
 public:
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
     return V->getValueID() == Value::FunctionVal ||
            V->getValueID() == Value::GlobalVariableVal;

llvm/include/llvm/IR/GlobalValue.h

enum ThreadLocalMode {
     GeneralDynamicTLSModel,
     LocalDynamicTLSModel,
     InitialExecTLSModel,
     LocalExecTLSModel
   };
 
   GlobalValue(const GlobalValue &) = delete;
 
-  unsigned getAlignment() const;
   unsigned getAddressSpace() const;
 
   enum class UnnamedAddr {
     None,
     Local,
     Global,
   };
 
/// @}
   ///
   /// NB! Ideally this should not be used at the IR level at all.  If you're
   /// interested in optimization constraints implied by the linker's ability to
   /// choose an implementation, prefer using \c hasExactDefinition.
   bool isStrongDefinitionForLinker() const {
     return !(isDeclarationForLinker() || isWeakForLinker());
   }
 
-  // Returns true if the alignment of the value can be unilaterally
-  // increased.
-  bool canIncreaseAlignment() const;
-
   const GlobalObject *getBaseObject() const;
   GlobalObject *getBaseObject() {
     return const_cast<GlobalObject *>(
                        static_cast<const GlobalValue *>(this)->getBaseObject());
   }
 
   /// Returns whether this is a reference to an absolute symbol.
   bool isAbsoluteSymbolRef() const;

llvm/lib/Bitcode/Writer/BitcodeWriter.cpp

writeStringRecord(Stream, bitc::MODULE_CODE_ASM, M.getModuleInlineAsm(),
                       0 /*TODO*/);
 
   // Emit information about sections and GC, computing how many there are. Also
   // compute the maximum alignment value.
   std::map<std::string, unsigned> SectionMap;
   std::map<std::string, unsigned> GCMap;
   unsigned MaxAlignment = 0;
   unsigned MaxGlobalType = 0;
-  for (const GlobalValue &GV : M.globals()) {
+  for (const GlobalVariable &GV : M.globals()) {
     MaxAlignment = std::max(MaxAlignment, GV.getAlignment());
     MaxGlobalType = std::max(MaxGlobalType, VE.getTypeID(GV.getValueType()));
     if (GV.hasSection()) {
       // Give section names unique ID's.
       unsigned &Entry = SectionMap[std::string(GV.getSection())];
       if (!Entry) {
         writeStringRecord(Stream, bitc::MODULE_CODE_SECTIONNAME, GV.getSection(),
                           0 /*TODO*/);

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

 static gcp_map_type &getGCMap(void *&P) {
   if (!P)
     P = new gcp_map_type();
   return *(gcp_map_type*)P;
 }
 
 /// getGVAlignment - Return the alignment to use for the specified global
 /// value.  This rounds up to the preferred alignment if possible and legal.
-Align AsmPrinter::getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
+Align AsmPrinter::getGVAlignment(const GlobalObject *GV, const DataLayout &DL,
                                  Align InAlign) {
   Align Alignment;
   if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
     Alignment = Align(DL.getPreferredAlignment(GVar));
 
   // If InAlign is specified, round it to it.
   if (InAlign > Alignment)
     Alignment = InAlign;

llvm/lib/IR/ConstantFold.cpp

case Instruction::And:
             // FIXME: This code should be deleted once existing targets have
             // appropriate defaults
             if (isa<Function>(GV) && !DL.getFunctionPtrAlign())
               GVAlign = Align(4);
           } else if (isa<Function>(GV)) {
             // Without a datalayout we have to assume the worst case: that the
             // function pointer isn't aligned at all.
             GVAlign = llvm::None;
-          } else {
-            GVAlign = MaybeAlign(GV->getAlignment());
+          } else if (isa<GlobalVariable>(GV)) {
+            GVAlign = cast<GlobalVariable>(GV)->getAlign();
           }
 
           if (GVAlign && *GVAlign > 1) {
             unsigned DstWidth = CI2->getType()->getBitWidth();
             unsigned SrcWidth = std::min(DstWidth, Log2(*GVAlign));
             APInt BitsNotSet(APInt::getLowBitsSet(DstWidth, SrcWidth));
 
             // If checking bits we know are clear, return zero.

llvm/lib/IR/Core.cpp

 LLVMTypeRef LLVMGlobalGetValueType(LLVMValueRef Global) {
   return wrap(unwrap<GlobalValue>(Global)->getValueType());
 }
 
 /*--.. Operations on global variables, load and store instructions .........--*/
 
 unsigned LLVMGetAlignment(LLVMValueRef V) {
   Value *P = unwrap<Value>(V);
-  if (GlobalValue *GV = dyn_cast<GlobalValue>(P))
+  if (GlobalObject *GV = dyn_cast<GlobalObject>(P))
     return GV->getAlignment();
   if (AllocaInst *AI = dyn_cast<AllocaInst>(P))
     return AI->getAlignment();
   if (LoadInst *LI = dyn_cast<LoadInst>(P))
     return LI->getAlignment();
   if (StoreInst *SI = dyn_cast<StoreInst>(P))
     return SI->getAlignment();
 
   llvm_unreachable(
-      "only GlobalValue, AllocaInst, LoadInst and StoreInst have alignment");
+      "only GlobalObject, AllocaInst, LoadInst and StoreInst have alignment");
 }
 
 void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) {
   Value *P = unwrap<Value>(V);
   if (GlobalObject *GV = dyn_cast<GlobalObject>(P))
     GV->setAlignment(MaybeAlign(Bytes));
   else if (AllocaInst *AI = dyn_cast<AllocaInst>(P))
     AI->setAlignment(Align(Bytes));

llvm/lib/IR/Globals.cpp

 
 bool GlobalValue::isInterposable() const {
   if (isInterposableLinkage(getLinkage()))
     return true;
   return getParent() && getParent()->getSemanticInterposition() &&
          !isDSOLocal();
 }
 
-unsigned GlobalValue::getAlignment() const {
-  if (auto *GA = dyn_cast<GlobalAlias>(this)) {
-    // In general we cannot compute this at the IR level, but we try.
-    if (const GlobalObject *GO = GA->getBaseObject())
-      return GO->getAlignment();
-
-    // FIXME: we should also be able to handle:
-    // Alias = Global + Offset
-    // Alias = Absolute
-    return 0;
-  }
-  return cast<GlobalObject>(this)->getAlignment();
-}
-
 unsigned GlobalValue::getAddressSpace() const {
   PointerType *PtrTy = getType();
   return PtrTy->getAddressSpace();
 }
 
 void GlobalObject::setAlignment(MaybeAlign Align) {
   assert((!Align || Align <= MaximumAlignment) &&
          "Alignment is greater than MaximumAlignment!");
bool GlobalValue::isDeclaration() const {
   if (const Function *F = dyn_cast<Function>(this))
     return F->empty() && !F->isMaterializable();
 
   // Aliases and ifuncs are always definitions.
   assert(isa<GlobalIndirectSymbol>(this));
   return false;
 }
 
-bool GlobalValue::canIncreaseAlignment() const {
+bool GlobalObject::canIncreaseAlignment() const {
   // Firstly, can only increase the alignment of a global if it
   // is a strong definition.
   if (!isStrongDefinitionForLinker())
     return false;
 
   // It also has to either not have a section defined, or, not have
   // alignment specified. (If it is assigned a section, the global
   // could be densely packed with other objects in the section, and

llvm/lib/IR/Verifier.cpp

for (const Value *TheNextUser : User->materialized_users())
     if (Callback(TheNextUser))
       forEachUser(TheNextUser, Visited, Callback);
 }
 
 void Verifier::visitGlobalValue(const GlobalValue &GV) {
   Assert(!GV.isDeclaration() || GV.hasValidDeclarationLinkage(),
          "Global is external, but doesn't have external or weak linkage!", &GV);
 
-  Assert(GV.getAlignment() <= Value::MaximumAlignment,
-         "huge alignment values are unsupported", &GV);
+  if (const GlobalObject *GO = dyn_cast<GlobalObject>(&GV))
+    Assert(GO->getAlignment() <= Value::MaximumAlignment,
+           "huge alignment values are unsupported", GO);
   Assert(!GV.hasAppendingLinkage() || isa<GlobalVariable>(GV),
          "Only global variables can have appending linkage!", &GV);
 
   if (GV.hasAppendingLinkage()) {
     const GlobalVariable *GVar = dyn_cast<GlobalVariable>(&GV);
     Assert(GVar && GVar->getValueType()->isArrayTy(),
            "Only global arrays can have appending linkage!", GVar);
   }

llvm/lib/LTO/LTOModule.cpp

 void LTOModule::addDefinedFunctionSymbol(StringRef Name, const Function *F) {
   // add to list of defined symbols
   addDefinedSymbol(Name, F, true);
 }
 
 void LTOModule::addDefinedSymbol(StringRef Name, const GlobalValue *def,
                                  bool isFunction) {
   // set alignment part log2() can have rounding errors
-  uint32_t align = def->getAlignment();
+  uint32_t align = 0;
+  const GlobalObject *go = dyn_cast<GlobalObject>(def);
+  if (go)
+    align = go->getAlignment();
   uint32_t attr = align ? countTrailingZeros(align) : 0;
 
   // set permissions part
   if (isFunction) {
     attr |= LTO_SYMBOL_PERMISSIONS_CODE;
   } else {
     const GlobalVariable *gv = dyn_cast<GlobalVariable>(def);
     if (gv && gv->isConstant())

llvm/lib/Object/IRSymtab.cpp

if (GV->isThreadLocal())
     Sym.Flags |= 1 << storage::Symbol::FB_tls;
   if (GV->hasGlobalUnnamedAddr())
     Sym.Flags |= 1 << storage::Symbol::FB_unnamed_addr;
   if (GV->canBeOmittedFromSymbolTable())
     Sym.Flags |= 1 << storage::Symbol::FB_may_omit;
   Sym.Flags |= unsigned(GV->getVisibility()) << storage::Symbol::FB_visibility;
 
   if (Flags & object::BasicSymbolRef::SF_Common) {
+    auto *GVar = dyn_cast<GlobalVariable>(GV);
+    if (!GVar)
+      return make_error<StringError>("Only variables can have common linkage!",
+                                     inconvertibleErrorCode());
     Uncommon().CommonSize = GV->getParent()->getDataLayout().getTypeAllocSize(
         GV->getType()->getElementType());
-    Uncommon().CommonAlign = GV->getAlignment();
+    Uncommon().CommonAlign = GVar->getAlignment();
   }
 
   const GlobalObject *Base = GV->getBaseObject();
   if (!Base)
     return make_error<StringError>("Unable to determine comdat of alias!",
                                    inconvertibleErrorCode());
   if (const Comdat *C = Base->getComdat()) {
     Expected<int> ComdatIndexOrErr = getComdatIndex(C, GV->getParent());

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
   if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
     GlobalAddressSDNode *GAN =
         dyn_cast<GlobalAddressSDNode>(N.getOperand(1).getNode());
     Base = N.getOperand(0);
     OffImm = N.getOperand(1);
     if (!GAN)
       return true;
 
-    if (GAN->getOffset() % Size == 0) {
-      const GlobalValue *GV = GAN->getGlobal();
-      unsigned Alignment = GV->getAlignment();
-      Type *Ty = GV->getValueType();
-      if (Alignment == 0 && Ty->isSized())
-        Alignment = DL.getABITypeAlignment(Ty);
-
-      if (Alignment >= Size)
-        return true;
-    }
-  }
+    if (GAN->getOffset() % Size == 0 &&
+        GAN->getGlobal()->getPointerAlignment(DL) >= Size)
 
   if (CurDAG->isBaseWithConstantOffset(N)) {
     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
       int64_t RHSC = (int64_t)RHS->getZExtValue();
       unsigned Scale = Log2_32(Size);
       if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
         Base = N.getOperand(0);
         if (Base.getOpcode() == ISD::FrameIndex) {

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

if (!MFI->isEntryFunction()) {
       SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode());
       SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                                         Trap, DAG.getRoot());
       DAG.setRoot(OutputChain);
       return DAG.getUNDEF(Op.getValueType());
     }
 
     // XXX: What does the value of G->getOffset() mean?
     assert(G->getOffset() == 0 &&
          "Do not know what to do with an non-zero offset");
 
     // TODO: We could emit code to handle the initialization somewhere.
     if (!hasDefinedInitializer(GV)) {
-      unsigned Offset = MFI->allocateLDSGlobal(DL, *GV);
+      unsigned Offset = MFI->allocateLDSGlobal(DL, *cast<GlobalVariable>(GV));
       return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
     }
   }
 
   const Function &Fn = DAG.getMachineFunction().getFunction();
   DiagnosticInfoUnsupported BadInit(
       Fn, "unsupported initializer for address space", SDLoc(Op).getDebugLoc());
   DAG.getContext()->diagnose(BadInit);
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
                                                   SelectionDAG &DAG) const {
   SmallVector<SDValue, 8> Args;
 
   EVT VT = Op.getValueType();
   if (VT == MVT::v4i16 || VT == MVT::v4f16) {
     SDLoc SL(Op);
     SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0));
     SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1));
 
     SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi });
     return DAG.getNode(ISD::BITCAST, SL, VT, BV);
   }
 
   for (const SDUse &U : Op->ops())
     DAG.ExtractVectorElements(U.get(), Args);
 
   return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
 }
 
 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
                                                      SelectionDAG &DAG) const {
 
   SmallVector<SDValue, 8> Args;
   unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
   EVT VT = Op.getValueType();
   DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
                             VT.getVectorNumElements());
 
   return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
 }
 
 /// Generate Min/Max node
 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
                                                    SDValue LHS, SDValue RHS,
                                                    SDValue True, SDValue False,
                                                    SDValue CC,
                                                    DAGCombinerInfo &DCI) const {
   if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
   switch (CCOpcode) {
   case ISD::SETOEQ:
   case ISD::SETONE:
   case ISD::SETUNE:
   case ISD::SETNE:
   case ISD::SETUEQ:
   case ISD::SETEQ:
   case ISD::SETFALSE:
   case ISD::SETFALSE2:
   case ISD::SETTRUE:
   case ISD::SETTRUE2:
   case ISD::SETUO:
   case ISD::SETO:
     break;
   case ISD::SETULE:
   case ISD::SETULT: {
     if (LHS == True)
       return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
     return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
   }
   case ISD::SETOLE:
   case ISD::SETOLT:
   case ISD::SETLE:
   case ISD::SETLT: {
     // Ordered. Assume ordered for undefined.
 
     // Only do this after legalization to avoid interfering with other combines
     // which might occur.
     if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
         !DCI.isCalledByLegalizer())
       return SDValue();
 
     // We need to permute the operands to get the correct NaN behavior. The
     // selected operand is the second one based on the failing compare with NaN,
     // so permute it based on the compare type the hardware uses.
     if (LHS == True)
       return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
     return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
   }
   case ISD::SETUGE:
   case ISD::SETUGT: {
     if (LHS == True)
       return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
     return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
   }
   case ISD::SETGT:
   case ISD::SETGE:
   case ISD::SETOGE:
   case ISD::SETOGT: {
     if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
         !DCI.isCalledByLegalizer())
       return SDValue();
 
     if (LHS == True)
       return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
     return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
   }
   case ISD::SETCC_INVALID:
     llvm_unreachable("Invalid setcc condcode!");
   }
   return SDValue();
 }
 
 std::pair<SDValue, SDValue>
 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
 
   SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
 
   const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
   const SDValue One = DAG.getConstant(1, SL, MVT::i32);
 
   SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
 
   return std::make_pair(Lo, Hi);
 }
 
 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
 
   SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
   const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
 }
 
 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
 
   SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
   const SDValue One = DAG.getConstant(1, SL, MVT::i32);
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
 }
 
 // Split a vector type into two parts. The first part is a power of two vector.
 // The second part is whatever is left over, and is a scalar if it would
 // otherwise be a 1-vector.
 std::pair<EVT, EVT>
 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
   EVT LoVT, HiVT;
   EVT EltVT = VT.getVectorElementType();
   unsigned NumElts = VT.getVectorNumElements();
   unsigned LoNumElts = PowerOf2Ceil((NumElts + 1) / 2);
   LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts);
   HiVT = NumElts - LoNumElts == 1
              ? EltVT
              : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts);
   return std::make_pair(LoVT, HiVT);
 }
 
 // Split a vector value into two parts of types LoVT and HiVT. HiVT could be
 // scalar.
 std::pair<SDValue, SDValue>
 AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
                                   const EVT &LoVT, const EVT &HiVT,
                                   SelectionDAG &DAG) const {
   assert(LoVT.getVectorNumElements() +
                  (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
              N.getValueType().getVectorNumElements() &&
          "More vector elements requested than available!");
   SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
                            DAG.getVectorIdxConstant(0, DL));
   SDValue Hi = DAG.getNode(
       HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL,
       HiVT, N, DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), DL));
   return std::make_pair(Lo, Hi);
 }
 
 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
                                               SelectionDAG &DAG) const {
   LoadSDNode *Load = cast<LoadSDNode>(Op);
   EVT VT = Op.getValueType();
   SDLoc SL(Op);
 
 
   // If this is a 2 element vector, we really want to scalarize and not create
   // weird 1 element vectors.
   if (VT.getVectorNumElements() == 2) {
     SDValue Ops[2];
     std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG);
     return DAG.getMergeValues(Ops, SL);
   }
 
   SDValue BasePtr = Load->getBasePtr();
   EVT MemVT = Load->getMemoryVT();
 
   const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
 
   EVT LoVT, HiVT;
   EVT LoMemVT, HiMemVT;
   SDValue Lo, Hi;
 
   std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
   std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
   std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG);
 
   unsigned Size = LoMemVT.getStoreSize();
   unsigned BaseAlign = Load->getAlignment();
   unsigned HiAlign = MinAlign(BaseAlign, Size);
 
   SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
                                   Load->getChain(), BasePtr, SrcValue, LoMemVT,
                                   BaseAlign, Load->getMemOperand()->getFlags());
   SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, Size);
   SDValue HiLoad =
       DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
                      HiPtr, SrcValue.getWithOffset(LoMemVT.getStoreSize()),
                      HiMemVT, HiAlign, Load->getMemOperand()->getFlags());
 
   SDValue Join;
   if (LoVT == HiVT) {
     // This is the case that the vector is power of two so was evenly split.
     Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
   } else {
     Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad,
                        DAG.getVectorIdxConstant(0, SL));
     Join = DAG.getNode(
         HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL,
         VT, Join, HiLoad,
         DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), SL));
   }
 
   SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
                                      LoLoad.getValue(1), HiLoad.getValue(1))};
 
   return DAG.getMergeValues(Ops, SL);
 }
 
 // Widen a vector load from vec3 to vec4.
 SDValue AMDGPUTargetLowering::WidenVectorLoad(SDValue Op,
                                               SelectionDAG &DAG) const {
   LoadSDNode *Load = cast<LoadSDNode>(Op);
   EVT VT = Op.getValueType();
   assert(VT.getVectorNumElements() == 3);
   SDValue BasePtr = Load->getBasePtr();
   EVT MemVT = Load->getMemoryVT();
   SDLoc SL(Op);
   const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
   unsigned BaseAlign = Load->getAlignment();
 
   EVT WideVT =
       EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
   EVT WideMemVT =
       EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 4);
   SDValue WideLoad = DAG.getExtLoad(
       Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue,
       WideMemVT, BaseAlign, Load->getMemOperand()->getFlags());
   return DAG.getMergeValues(
       {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad,
                    DAG.getVectorIdxConstant(0, SL)),
        WideLoad.getValue(1)},
       SL);
 }
 
 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
                                                SelectionDAG &DAG) const {
   StoreSDNode *Store = cast<StoreSDNode>(Op);
   SDValue Val = Store->getValue();
   EVT VT = Val.getValueType();
 
   // If this is a 2 element vector, we really want to scalarize and not create
   // weird 1 element vectors.
   if (VT.getVectorNumElements() == 2)
     return scalarizeVectorStore(Store, DAG);
 
   EVT MemVT = Store->getMemoryVT();
   SDValue Chain = Store->getChain();
   SDValue BasePtr = Store->getBasePtr();
   SDLoc SL(Op);
 
   EVT LoVT, HiVT;
   EVT LoMemVT, HiMemVT;
   SDValue Lo, Hi;
 
   std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
   std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
   std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG);
 
   SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());
 
   const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
   unsigned BaseAlign = Store->getAlignment();
   unsigned Size = LoMemVT.getStoreSize();
   unsigned HiAlign = MinAlign(BaseAlign, Size);
 
   SDValue LoStore =
       DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
                         Store->getMemOperand()->getFlags());
   SDValue HiStore =
       DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size),
                         HiMemVT, HiAlign, Store->getMemOperand()->getFlags());
 
   return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
 }
 
 // This is a shortcut for integer division because we have fast i32<->f32
 // conversions, and fast f32 reciprocal instructions. The fractional part of a
 // float is enough to accurately represent up to a 24-bit signed integer.
 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
                                             bool Sign) const {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
   MVT IntVT = MVT::i32;
   MVT FltVT = MVT::f32;
 
   unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS);
   if (LHSSignBits < 9)
     return SDValue();
 
   unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS);
   if (RHSSignBits < 9)
     return SDValue();
 
   unsigned BitSize = VT.getSizeInBits();
   unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
   unsigned DivBits = BitSize - SignBits;
   if (Sign)
     ++DivBits;
 
   ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
   ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
 
   SDValue jq = DAG.getConstant(1, DL, IntVT);
 
   if (Sign) {
     // char|short jq = ia ^ ib;
     jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
 
     // jq = jq >> (bitsize - 2)
     jq = DAG.getNode(ISD::SRA, DL, VT, jq,
                      DAG.getConstant(BitSize - 2, DL, VT));
 
     // jq = jq | 0x1
     jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
   }
 
   // int ia = (int)LHS;
   SDValue ia = LHS;
 
   // int ib, (int)RHS;
   SDValue ib = RHS;
 
   // float fa = (float)ia;
   SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
 
   // float fb = (float)ib;
   SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
 
   SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
                            fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
 
   // fq = trunc(fq);
   fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
 
   // float fqneg = -fq;
   SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
 
   MachineFunction &MF = DAG.getMachineFunction();
   const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
 
   // float fr = mad(fqneg, fb, fa);
   unsigned OpCode = !MFI->getMode().allFP32Denormals() ?
                     (unsigned)ISD::FMAD :
                     (unsigned)AMDGPUISD::FMAD_FTZ;
 
   SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
 
   // int iq = (int)fq;
   SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
 
   // fr = fabs(fr);
   fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
 
   // fb = fabs(fb);
   fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
 
   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
 
   // int cv = fr >= fb;
   SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
 
   // jq = (cv ? jq : 0);
   jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
 
   // dst = iq + jq;
   SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
 
   // Rem needs compensation, it's easier to recompute it
   SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
   Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
 
   // Truncate to number of bits this divide really is.
   if (Sign) {
     SDValue InRegSize
       = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits));
     Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
     Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
   } else {
     SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
     Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
     Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
   }
 
   return DAG.getMergeValues({ Div, Rem }, DL);
 }
 
 void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
                                       SelectionDAG &DAG,
                                       SmallVectorImpl<SDValue> &Results) const {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
 
   EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
 
   SDValue One = DAG.getConstant(1, DL, HalfVT);
   SDValue Zero = DAG.getConstant(0, DL, HalfVT);
 
   //HiLo split
   SDValue LHS = Op.getOperand(0);
   SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
   SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One);
 
   SDValue RHS = Op.getOperand(1);
   SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
   SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One);
 
   if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
       DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {
 
     SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
                               LHS_Lo, RHS_Lo);
 
     SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
     SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
 
     Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
     Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
     return;
   }
 
   if (isTypeLegal(MVT::i64)) {
     MachineFunction &MF = DAG.getMachineFunction();
     const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
     // Compute denominator reciprocal.
     unsigned FMAD = !MFI->getMode().allFP32Denormals() ?
                     (unsigned)ISD::FMAD :
                     (unsigned)AMDGPUISD::FMAD_FTZ;
 
 
     SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
     SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
     SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
       DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
       Cvt_Lo);
     SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
     SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
       DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
     SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
       DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
     SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
     SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
       DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
       Mul1);
     SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
     SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
     SDValue Rcp64 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
 
     SDValue Zero64 = DAG.getConstant(0, DL, VT);
     SDValue One64  = DAG.getConstant(1, DL, VT);
     SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
     SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);
 
     SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
     SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
     SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
     SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
                                     Zero);
     SDValue Mulhi1_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
                                     One);
 
     SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
                                   Mulhi1_Lo, Zero1);
     SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
                                   Mulhi1_Hi, Add1_Lo.getValue(1));
     SDValue Add1_HiNc = DAG.getNode(ISD::ADD, DL, HalfVT, Rcp_Hi, Mulhi1_Hi);
     SDValue Add1 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
 
     SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
     SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
     SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
                                     Zero);
     SDValue Mulhi2_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
                                     One);
 
     SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
                                   Mulhi2_Lo, Zero1);
     SDValue Add2_HiC = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_HiNc,
                                    Mulhi2_Hi, Add1_Lo.getValue(1));
     SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add2_HiC,
                                   Zero, Add2_Lo.getValue(1));
     SDValue Add2 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
     SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
 
     SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
 
     SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero);
     SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One);
     SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
                                   Mul3_Lo, Zero1);
     SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
                                   Mul3_Hi, Sub1_Lo.getValue(1));
     SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
     SDValue Sub1 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
 
     SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
     SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
                                  ISD::SETUGE);
     SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
                                  ISD::SETUGE);
     SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
 
     // TODO: Here and below portions of the code can be enclosed into if/endif.
     // Currently control flow is unconditional and we have 4 selects after
     // potential endif to substitute PHIs.
 
     // if C3 != 0 ...
     SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
                                   RHS_Lo, Zero1);
     SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
                                   RHS_Hi, Sub1_Lo.getValue(1));
     SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
                                   Zero, Sub2_Lo.getValue(1));
     SDValue Sub2 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
 
     SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
 
     SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
                                  ISD::SETUGE);
     SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
                                  ISD::SETUGE);
     SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
 
     // if (C6 != 0)
     SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
 
     SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
                                   RHS_Lo, Zero1);
     SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
                                   RHS_Hi, Sub2_Lo.getValue(1));
     SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
                                   Zero, Sub3_Lo.getValue(1));
     SDValue Sub3 = DAG.getBitcast(VT,
                         DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
 
     // endif C6
     // endif C3
 
     SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
     SDValue Div  = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
 
     SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
     SDValue Rem  = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
 
     Results.push_back(Div);
     Results.push_back(Rem);
 
     return;
   }
 
   // r600 expandion.
   // Get Speculative values
   SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
   SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
 
   SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
   SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
   REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
 
   SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
   SDValue DIV_Lo = Zero;
 
   const unsigned halfBitWidth = HalfVT.getSizeInBits();
 
   for (unsigned i = 0; i < halfBitWidth; ++i) {
     const unsigned bitPos = halfBitWidth - i - 1;
     SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
     // Get value of high bit
     SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
     HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One);
     HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
 
     // Shift
     REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
     // Add LHS high bit
     REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
 
     SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
     SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
 
     DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
 
     // Update REM
     SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
     REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
   }
 
   SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
   DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
   Results.push_back(DIV);
   Results.push_back(REM);
 }
 
 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   if (VT == MVT::i64) {
     SmallVector<SDValue, 2> Results;
     LowerUDIVREM64(Op, DAG, Results);
     return DAG.getMergeValues(Results, DL);
   }
 
   if (VT == MVT::i32) {
     if (SDValue Res = LowerDIVREM24(Op, DAG, false))
       return Res;
   }
 
   SDValue Num = Op.getOperand(0);
   SDValue Den = Op.getOperand(1);
 
   // RCP =  URECIP(Den) = 2^32 / Den + e
   // e is rounding error.
   SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
 
   // RCP_LO = mul(RCP, Den) */
   SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den);
 
   // RCP_HI = mulhu (RCP, Den) */
   SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
 
   // NEG_RCP_LO = -RCP_LO
   SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
                                                      RCP_LO);
 
   // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
   SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
                                            NEG_RCP_LO, RCP_LO,
                                            ISD::SETEQ);
   // Calculate the rounding error from the URECIP instruction
   // E = mulhu(ABS_RCP_LO, RCP)
   SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
 
   // RCP_A_E = RCP + E
   SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
 
   // RCP_S_E = RCP - E
   SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
 
   // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
   SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
                                      RCP_A_E, RCP_S_E,
                                      ISD::SETEQ);
   // Quotient = mulhu(Tmp0, Num)
   SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
 
   // Num_S_Remainder = Quotient * Den
   SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den);
 
   // Remainder = Num - Num_S_Remainder
   SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
 
   // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
   SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
                                                  DAG.getConstant(-1, DL, VT),
                                                  DAG.getConstant(0, DL, VT),
                                                  ISD::SETUGE);
   // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
   SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
                                                   Num_S_Remainder,
                                                   DAG.getConstant(-1, DL, VT),
                                                   DAG.getConstant(0, DL, VT),
                                                   ISD::SETUGE);
   // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
   SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
                                                Remainder_GE_Zero);
 
   // Calculate Division result:
 
   // Quotient_A_One = Quotient + 1
   SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
                                        DAG.getConstant(1, DL, VT));
 
   // Quotient_S_One = Quotient - 1
   SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
                                        DAG.getConstant(1, DL, VT));
 
   // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
   SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
                                      Quotient, Quotient_A_One, ISD::SETEQ);
 
   // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
   Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
                             Quotient_S_One, Div, ISD::SETEQ);
 
   // Calculate Rem result:
 
   // Remainder_S_Den = Remainder - Den
   SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
 
   // Remainder_A_Den = Remainder + Den
   SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
 
   // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
   SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
                                     Remainder, Remainder_S_Den, ISD::SETEQ);
 
   // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
   Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
                             Remainder_A_Den, Rem, ISD::SETEQ);
   SDValue Ops[2] = {
     Div,
     Rem
   };
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
 
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue NegOne = DAG.getConstant(-1, DL, VT);
 
   if (VT == MVT::i32) {
     if (SDValue Res = LowerDIVREM24(Op, DAG, true))
       return Res;
   }
 
   if (VT == MVT::i64 &&
       DAG.ComputeNumSignBits(LHS) > 32 &&
       DAG.ComputeNumSignBits(RHS) > 32) {
     EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
 
     //HiLo split
     SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
     SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
     SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
                                  LHS_Lo, RHS_Lo);
     SDValue Res[2] = {
       DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
       DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
     };
     return DAG.getMergeValues(Res, DL);
   }
 
   SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
   SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
   SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
   SDValue RSign = LHSign; // Remainder sign is the same as LHS
 
   LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
   RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
 
   LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
   RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
 
   SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
   SDValue Rem = Div.getValue(1);
 
   Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
   Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
 
   Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
   Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
 
   SDValue Res[2] = {
     Div,
     Rem
   };
   return DAG.getMergeValues(Res, DL);
 }
 
 // (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y))
 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   EVT VT = Op.getValueType();
   SDValue X = Op.getOperand(0);
   SDValue Y = Op.getOperand(1);
 
   // TODO: Should this propagate fast-math-flags?
 
   SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
   SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
   SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
 
   return DAG.getNode(ISD::FSUB, SL, VT, X, Mul);
 }
 
 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   // result = trunc(src)
   // if (src > 0.0 && src != result)
   //   result += 1.0
 
   SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
 
   const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
   const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
 
   EVT SetCCVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
 
   SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
   SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
   SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
 
   SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
   // TODO: Should this propagate fast-math-flags?
   return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
 }
 
 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
                                   SelectionDAG &DAG) {
   const unsigned FractBits = 52;
   const unsigned ExpBits = 11;
 
   SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
                                 Hi,
                                 DAG.getConstant(FractBits - 32, SL, MVT::i32),
                                 DAG.getConstant(ExpBits, SL, MVT::i32));
   SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
                             DAG.getConstant(1023, SL, MVT::i32));
 
   return Exp;
 }
 
 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   assert(Op.getValueType() == MVT::f64);
 
   const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
   const SDValue One = DAG.getConstant(1, SL, MVT::i32);
 
   SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
 
   // Extract the upper half, since this is where we will find the sign and
   // exponent.
   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
 
   SDValue Exp = extractF64Exponent(Hi, SL, DAG);
 
   const unsigned FractBits = 52;
 
   // Extract the sign bit.
   const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
   SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
 
   // Extend back to 64-bits.
   SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
   SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
 
   SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
   const SDValue FractMask
     = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
 
   SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
   SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
   SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
 
   EVT SetCCVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
 
   const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
 
   SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
   SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
 
   SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
   SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
 
   return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
 }
 
 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   assert(Op.getValueType() == MVT::f64);
 
   APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
   SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
   SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
 
   // TODO: Should this propagate fast-math-flags?
 
   SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
   SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
 
   SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
 
   APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
   SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
 
   EVT SetCCVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
   SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
 
   return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
 }
 
 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
   // FNEARBYINT and FRINT are the same, except in their handling of FP
   // exceptions. Those aren't really meaningful for us, and OpenCL only has
   // rint, so just treat them as equivalent.
   return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0));
 }
 
 // XXX - May require not supporting f32 denormals?
 
 // Don't handle v2f16. The extra instructions to scalarize and repack around the
 // compare and vselect end up producing worse code than scalarizing the whole
 // operation.
 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue X = Op.getOperand(0);
   EVT VT = Op.getValueType();
 
   SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
 
   // TODO: Should this propagate fast-math-flags?
 
   SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
 
   SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
 
   const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
   const SDValue One = DAG.getConstantFP(1.0, SL, VT);
   const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
 
   SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);
 
   EVT SetCCVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
 
   SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
 
   SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);
 
   return DAG.getNode(ISD::FADD, SL, VT, T, Sel);
 }
 
 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   // result = trunc(src);
   // if (src < 0.0 && src != result)
   //   result += -1.0.
 
   SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
 
   const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
   const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
 
   EVT SetCCVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
 
   SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
   SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
   SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
 
   SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
   // TODO: Should this propagate fast-math-flags?
   return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
 }
 
 SDValue AMDGPUTargetLowering::LowerFLOG(SDValue Op, SelectionDAG &DAG,
                                         double Log2BaseInverted) const {
   EVT VT = Op.getValueType();
 
   SDLoc SL(Op);
   SDValue Operand = Op.getOperand(0);
   SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand);
   SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
 
   return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand);
 }
 
 // exp2(M_LOG2E_F * f);
 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   const SDValue K = DAG.getConstantFP(numbers::log2e, SL, VT);
   SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags());
   return DAG.getNode(ISD::FEXP2, SL, VT, Mul, Op->getFlags());
 }
 
 static bool isCtlzOpc(unsigned Opc) {
   return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
 }
 
 static bool isCttzOpc(unsigned Opc) {
   return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF;
 }
 
 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
   bool ZeroUndef = Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
                    Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF;
 
   unsigned ISDOpc, NewOpc;
   if (isCtlzOpc(Op.getOpcode())) {
     ISDOpc = ISD::CTLZ_ZERO_UNDEF;
     NewOpc = AMDGPUISD::FFBH_U32;
   } else if (isCttzOpc(Op.getOpcode())) {
     ISDOpc = ISD::CTTZ_ZERO_UNDEF;
     NewOpc = AMDGPUISD::FFBL_B32;
   } else
     llvm_unreachable("Unexpected OPCode!!!");
 
 
   if (ZeroUndef && Src.getValueType() == MVT::i32)
     return DAG.getNode(NewOpc, SL, MVT::i32, Src);
 
   SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
 
   const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
   const SDValue One = DAG.getConstant(1, SL, MVT::i32);
 
   SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
 
   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
                                    *DAG.getContext(), MVT::i32);
 
   SDValue HiOrLo = isCtlzOpc(Op.getOpcode()) ? Hi : Lo;
   SDValue Hi0orLo0 = DAG.getSetCC(SL, SetCCVT, HiOrLo, Zero, ISD::SETEQ);
 
   SDValue OprLo = DAG.getNode(ISDOpc, SL, MVT::i32, Lo);
   SDValue OprHi = DAG.getNode(ISDOpc, SL, MVT::i32, Hi);
 
   const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32);
   SDValue Add, NewOpr;
   if (isCtlzOpc(Op.getOpcode())) {
     Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprLo, Bits32);
     // ctlz(x) = hi_32(x) == 0 ? ctlz(lo_32(x)) + 32 : ctlz(hi_32(x))
     NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprHi);
   } else {
     Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprHi, Bits32);
     // cttz(x) = lo_32(x) == 0 ? cttz(hi_32(x)) + 32 : cttz(lo_32(x))
     NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprLo);
   }
 
   if (!ZeroUndef) {
     // Test if the full 64-bit input is zero.
 
     // FIXME: DAG combines turn what should be an s_and_b64 into a v_or_b32,
     // which we probably don't want.
     SDValue LoOrHi = isCtlzOpc(Op.getOpcode()) ? Lo : Hi;
     SDValue Lo0OrHi0 = DAG.getSetCC(SL, SetCCVT, LoOrHi, Zero, ISD::SETEQ);
     SDValue SrcIsZero = DAG.getNode(ISD::AND, SL, SetCCVT, Lo0OrHi0, Hi0orLo0);
 
     // TODO: If i64 setcc is half rate, it can result in 1 fewer instruction
     // with the same cycles, otherwise it is slower.
     // SDValue SrcIsZero = DAG.getSetCC(SL, SetCCVT, Src,
     // DAG.getConstant(0, SL, MVT::i64), ISD::SETEQ);
 
     const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32);
 
     // The instruction returns -1 for 0 input, but the defined intrinsic
     // behavior is to return the number of bits.
     NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32,
                          SrcIsZero, Bits32, NewOpr);
   }
 
   return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
 }
 
 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
                                                bool Signed) const {
   // Unsigned
   // cul2f(ulong u)
   //{
   //  uint lz = clz(u);
   //  uint e = (u != 0) ? 127U + 63U - lz : 0;
   //  u = (u << lz) & 0x7fffffffffffffffUL;
   //  ulong t = u & 0xffffffffffUL;
   //  uint v = (e << 23) | (uint)(u >> 40);
   //  uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
   //  return as_float(v + r);
   //}
   // Signed
   // cl2f(long l)
   //{
   //  long s = l >> 63;
   //  float r = cul2f((l + s) ^ s);
   //  return s ? -r : r;
   //}
 
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
   SDValue L = Src;
 
   SDValue S;
   if (Signed) {
     const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64);
     S = DAG.getNode(ISD::SRA, SL, MVT::i64, L, SignBit);
 
     SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S);
     L = DAG.getNode(ISD::XOR, SL, MVT::i64, LPlusS, S);
   }
 
   EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
                                    *DAG.getContext(), MVT::f32);
 
 
   SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32);
   SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64);
   SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L);
   LZ = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LZ);
 
   SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32);
   SDValue E = DAG.getSelect(SL, MVT::i32,
     DAG.getSetCC(SL, SetCCVT, L, ZeroI64, ISD::SETNE),
     DAG.getNode(ISD::SUB, SL, MVT::i32, K, LZ),
     ZeroI32);
 
   SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64,
     DAG.getNode(ISD::SHL, SL, MVT::i64, L, LZ),
     DAG.getConstant((-1ULL) >> 1, SL, MVT::i64));
 
   SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U,
                           DAG.getConstant(0xffffffffffULL, SL, MVT::i64));
 
   SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64,
                              U, DAG.getConstant(40, SL, MVT::i64));
 
   SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32,
     DAG.getNode(ISD::SHL, SL, MVT::i32, E, DAG.getConstant(23, SL, MVT::i32)),
     DAG.getNode(ISD::TRUNCATE, SL, MVT::i32,  UShl));
 
   SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64);
   SDValue RCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETUGT);
   SDValue TCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETEQ);
 
   SDValue One = DAG.getConstant(1, SL, MVT::i32);
 
   SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One);
 
   SDValue R = DAG.getSelect(SL, MVT::i32,
     RCmp,
     One,
     DAG.getSelect(SL, MVT::i32, TCmp, VTrunc1, ZeroI32));
   R = DAG.getNode(ISD::ADD, SL, MVT::i32, V, R);
   R = DAG.getNode(ISD::BITCAST, SL, MVT::f32, R);
 
   if (!Signed)
     return R;
 
   SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R);
   return DAG.getSelect(SL, MVT::f32, DAG.getSExtOrTrunc(S, SL, SetCCVT), RNeg, R);
 }
 
 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
                                                bool Signed) const {
   SDLoc SL(Op);
   SDValue Src = Op.getOperand(0);
 
   SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
 
   SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
                            DAG.getConstant(0, SL, MVT::i32));
   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
                            DAG.getConstant(1, SL, MVT::i32));
 
   SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
                               SL, MVT::f64, Hi);
 
   SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
 
   SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
                               DAG.getConstant(32, SL, MVT::i32));
   // TODO: Should this propagate fast-math-flags?
   return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
 }
 
 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
                                                SelectionDAG &DAG) const {
   // TODO: Factor out code common with LowerSINT_TO_FP.
   EVT DestVT = Op.getValueType();
   SDValue Src = Op.getOperand(0);
   EVT SrcVT = Src.getValueType();
 
   if (SrcVT == MVT::i16) {
     if (DestVT == MVT::f16)
       return Op;
     SDLoc DL(Op);
 
     // Promote src to i32
     SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src);
     return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext);
   }
 
   assert(SrcVT == MVT::i64 && "operation should be legal");
 
   if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
     SDLoc DL(Op);
 
     SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
     SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
     SDValue FPRound =
         DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
 
     return FPRound;
   }
 
   if (DestVT == MVT::f32)
     return LowerINT_TO_FP32(Op, DAG, false);
 
   assert(DestVT == MVT::f64);
   return LowerINT_TO_FP64(Op, DAG, false);
 }
 
 SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
                                               SelectionDAG &DAG) const {
   EVT DestVT = Op.getValueType();
 
   SDValue Src = Op.getOperand(0);
   EVT SrcVT = Src.getValueType();
 
   if (SrcVT == MVT::i16) {
     if (DestVT == MVT::f16)
       return Op;
 
     SDLoc DL(Op);
     // Promote src to i32
     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src);
     return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext);
   }
 
   assert(SrcVT == MVT::i64 && "operation should be legal");
 
   // TODO: Factor out code common with LowerUINT_TO_FP.
 
   if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
     SDLoc DL(Op);
     SDValue Src = Op.getOperand(0);
 
     SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
     SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
     SDValue FPRound =
         DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
 
     return FPRound;
   }
 
   if (DestVT == MVT::f32)
     return LowerINT_TO_FP32(Op, DAG, true);
 
   assert(DestVT == MVT::f64);
   return LowerINT_TO_FP64(Op, DAG, true);
 }
 
 SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
                                                bool Signed) const {
   SDLoc SL(Op);
 
   SDValue Src = Op.getOperand(0);
 
   SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
 
   SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
                                  MVT::f64);
   SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
                                  MVT::f64);
   // TODO: Should this propagate fast-math-flags?
   SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
 
   SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul);
 
 
   SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc);
 
   SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL,
                            MVT::i32, FloorMul);
   SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
 
   SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi});
 
   return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result);
 }
 
 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
   SDValue N0 = Op.getOperand(0);
 
   // Convert to target node to get known bits
   if (N0.getValueType() == MVT::f32)
     return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);
 
   if (getTargetMachine().Options.UnsafeFPMath) {
     // There is a generic expand for FP_TO_FP16 with unsafe fast math.
     return SDValue();
   }
 
   assert(N0.getSimpleValueType() == MVT::f64);
 
   // f64 -> f16 conversion using round-to-nearest-even rounding mode.
   const unsigned ExpMask = 0x7ff;
   const unsigned ExpBiasf64 = 1023;
   const unsigned ExpBiasf16 = 15;
   SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
   SDValue One = DAG.getConstant(1, DL, MVT::i32);
   SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
   SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
                            DAG.getConstant(32, DL, MVT::i64));
   UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
   U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
   SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
                           DAG.getConstant(20, DL, MVT::i64));
   E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
                   DAG.getConstant(ExpMask, DL, MVT::i32));
   // Subtract the fp64 exponent bias (1023) to get the real exponent and
   // add the f16 bias (15) to get the biased exponent for the f16 format.
   E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
                   DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
 
   SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
                           DAG.getConstant(8, DL, MVT::i32));
   M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
                   DAG.getConstant(0xffe, DL, MVT::i32));
 
   SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
                                   DAG.getConstant(0x1ff, DL, MVT::i32));
   MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);
 
   SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
   M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);
 
   // (M != 0 ? 0x0200 : 0) | 0x7c00;
   SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
       DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
                       Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
 
   // N = M | (E << 12);
   SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
       DAG.getNode(ISD::SHL, DL, MVT::i32, E,
                   DAG.getConstant(12, DL, MVT::i32)));
 
   // B = clamp(1-E, 0, 13);
   SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
                                   One, E);
   SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
   B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
                   DAG.getConstant(13, DL, MVT::i32));
 
   SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
                                    DAG.getConstant(0x1000, DL, MVT::i32));
 
   SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
   SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
   SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
   D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);
 
   SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
   SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
                               DAG.getConstant(0x7, DL, MVT::i32));
   V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
                   DAG.getConstant(2, DL, MVT::i32));
   SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
                                One, Zero, ISD::SETEQ);
   SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
                                One, Zero, ISD::SETGT);
   V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
   V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
 
   V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
                       DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
   V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
                       I, V, ISD::SETEQ);
 
   // Extract the sign bit.
   SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
                             DAG.getConstant(16, DL, MVT::i32));
   Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
                      DAG.getConstant(0x8000, DL, MVT::i32));
 
   V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
   return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
 }
 
 SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
 
   // TODO: Factor out code common with LowerFP_TO_UINT.
 
   EVT SrcVT = Src.getValueType();
   if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
     SDLoc DL(Op);
 
     SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
     SDValue FpToInt32 =
         DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
 
     return FpToInt32;
   }
 
   if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
     return LowerFP64_TO_INT(Op, DAG, true);
 
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
 
   // TODO: Factor out code common with LowerFP_TO_SINT.
 
   EVT SrcVT = Src.getValueType();
   if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
     SDLoc DL(Op);
 
     SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
     SDValue FpToInt32 =
         DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
 
     return FpToInt32;
   }
 
   if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
     return LowerFP64_TO_INT(Op, DAG, false);
 
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
                                                      SelectionDAG &DAG) const {
   EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
   MVT VT = Op.getSimpleValueType();
   MVT ScalarVT = VT.getScalarType();
 
   assert(VT.isVector());
 
   SDValue Src = Op.getOperand(0);
   SDLoc DL(Op);
 
   // TODO: Don't scalarize on Evergreen?
   unsigned NElts = VT.getVectorNumElements();
   SmallVector<SDValue, 8> Args;
   DAG.ExtractVectorElements(Src, Args, 0, NElts);
 
   SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
   for (unsigned I = 0; I < NElts; ++I)
     Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
 
   return DAG.getBuildVector(VT, DL, Args);
 }
 
 //===----------------------------------------------------------------------===//
 // Custom DAG optimizations
 //===----------------------------------------------------------------------===//
 
 static bool isU24(SDValue Op, SelectionDAG &DAG) {
   return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
 }
 
 static bool isI24(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
   return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
                                      // as unsigned 24-bit values.
     AMDGPUTargetLowering::numBitsSigned(Op, DAG) < 24;
 }
 
 static SDValue simplifyI24(SDNode *Node24,
                            TargetLowering::DAGCombinerInfo &DCI) {
   SelectionDAG &DAG = DCI.DAG;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
 
   SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0);
   SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1);
   unsigned NewOpcode = Node24->getOpcode();
   if (IsIntrin) {
     unsigned IID = cast<ConstantSDNode>(Node24->getOperand(0))->getZExtValue();
     NewOpcode = IID == Intrinsic::amdgcn_mul_i24 ?
       AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
   }
 
   APInt Demanded = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 24);
 
   // First try to simplify using SimplifyMultipleUseDemandedBits which allows
   // the operands to have other uses, but will only perform simplifications that
   // involve bypassing some nodes for this user.
   SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG);
   SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG);
   if (DemandedLHS || DemandedRHS)
     return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(),
                        DemandedLHS ? DemandedLHS : LHS,
                        DemandedRHS ? DemandedRHS : RHS);
 
   // Now try SimplifyDemandedBits which can simplify the nodes used by our
   // operands if this node is the only user.
   if (TLI.SimplifyDemandedBits(LHS, Demanded, DCI))
     return SDValue(Node24, 0);
   if (TLI.SimplifyDemandedBits(RHS, Demanded, DCI))
     return SDValue(Node24, 0);
 
   return SDValue();
 }
 
 template <typename IntTy>
 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
                                uint32_t Width, const SDLoc &DL) {
   if (Width + Offset < 32) {
     uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
     IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
     return DAG.getConstant(Result, DL, MVT::i32);
   }
 
   return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
 }
 
 static bool hasVolatileUser(SDNode *Val) {
   for (SDNode *U : Val->uses()) {
     if (MemSDNode *M = dyn_cast<MemSDNode>(U)) {
       if (M->isVolatile())
         return true;
     }
   }
 
   return false;
 }
 
 bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
   // i32 vectors are the canonical memory type.
   if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
     return false;
 
   if (!VT.isByteSized())
     return false;
 
   unsigned Size = VT.getStoreSize();
 
   if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
     return false;
 
   if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
     return false;
 
   return true;
 }
 
 // Replace load of an illegal type with a store of a bitcast to a friendlier
 // type.
 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   if (!DCI.isBeforeLegalize())
     return SDValue();
 
   LoadSDNode *LN = cast<LoadSDNode>(N);
   if (LN->isVolatile() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN))
     return SDValue();
 
   SDLoc SL(N);
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = LN->getMemoryVT();
 
   unsigned Size = VT.getStoreSize();
   unsigned Align = LN->getAlignment();
   if (Align < Size && isTypeLegal(VT)) {
     bool IsFast;
     unsigned AS = LN->getAddressSpace();
 
     // Expand unaligned loads earlier than legalization. Due to visitation order
     // problems during legalization, the emitted instructions to pack and unpack
     // the bytes again are not eliminated in the case of an unaligned copy.
     if (!allowsMisalignedMemoryAccesses(
             VT, AS, Align, LN->getMemOperand()->getFlags(), &IsFast)) {
       SDValue Ops[2];
 
       if (VT.isVector())
         std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(LN, DAG);
       else
         std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);
 
       return DAG.getMergeValues(Ops, SDLoc(N));
     }
 
     if (!IsFast)
       return SDValue();
   }
 
   if (!shouldCombineMemoryType(VT))
     return SDValue();
 
   EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
 
   SDValue NewLoad
     = DAG.getLoad(NewVT, SL, LN->getChain(),
                   LN->getBasePtr(), LN->getMemOperand());
 
   SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
   DCI.CombineTo(N, BC, NewLoad.getValue(1));
   return SDValue(N, 0);
 }
 
 // Replace store of an illegal type with a store of a bitcast to a friendlier
 // type.
 SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
                                                   DAGCombinerInfo &DCI) const {
   if (!DCI.isBeforeLegalize())
     return SDValue();
 
   StoreSDNode *SN = cast<StoreSDNode>(N);
   if (SN->isVolatile() || !ISD::isNormalStore(SN))
     return SDValue();
 
   EVT VT = SN->getMemoryVT();
   unsigned Size = VT.getStoreSize();
 
   SDLoc SL(N);
   SelectionDAG &DAG = DCI.DAG;
   unsigned Align = SN->getAlignment();
   if (Align < Size && isTypeLegal(VT)) {
     bool IsFast;
     unsigned AS = SN->getAddressSpace();
 
     // Expand unaligned stores earlier than legalization. Due to visitation
     // order problems during legalization, the emitted instructions to pack and
     // unpack the bytes again are not eliminated in the case of an unaligned
     // copy.
     if (!allowsMisalignedMemoryAccesses(
             VT, AS, Align, SN->getMemOperand()->getFlags(), &IsFast)) {
       if (VT.isVector())
         return scalarizeVectorStore(SN, DAG);
 
       return expandUnalignedStore(SN, DAG);
     }
 
     if (!IsFast)
       return SDValue();
   }
 
   if (!shouldCombineMemoryType(VT))
     return SDValue();
 
   EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
   SDValue Val = SN->getValue();
 
   //DCI.AddToWorklist(Val.getNode());
 
   bool OtherUses = !Val.hasOneUse();
   SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
   if (OtherUses) {
     SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
     DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
   }
 
   return DAG.getStore(SN->getChain(), SL, CastVal,
                       SN->getBasePtr(), SN->getMemOperand());
 }
 
 // FIXME: This should go in generic DAG combiner with an isTruncateFree check,
 // but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
 // issues.
 SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
                                                         DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
 
   // (vt2 (assertzext (truncate vt0:x), vt1)) ->
   //     (vt2 (truncate (assertzext vt0:x, vt1)))
   if (N0.getOpcode() == ISD::TRUNCATE) {
     SDValue N1 = N->getOperand(1);
     EVT ExtVT = cast<VTSDNode>(N1)->getVT();
     SDLoc SL(N);
 
     SDValue Src = N0.getOperand(0);
     EVT SrcVT = Src.getValueType();
     if (SrcVT.bitsGE(ExtVT)) {
       SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
       return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg);
     }
   }
 
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
   SDNode *N, DAGCombinerInfo &DCI) const {
   unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
   switch (IID) {
   case Intrinsic::amdgcn_mul_i24:
   case Intrinsic::amdgcn_mul_u24:
     return simplifyI24(N, DCI);
   case Intrinsic::amdgcn_fract:
   case Intrinsic::amdgcn_rsq:
   case Intrinsic::amdgcn_rcp_legacy:
   case Intrinsic::amdgcn_rsq_legacy:
   case Intrinsic::amdgcn_rsq_clamp:
   case Intrinsic::amdgcn_ldexp: {
     // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
     SDValue Src = N->getOperand(1);
     return Src.isUndef() ? Src : SDValue();
   }
   default:
     return SDValue();
   }
 }
 
 /// Split the 64-bit value \p LHS into two 32-bit components, and perform the
 /// binary operation \p Opc to it with the corresponding constant operands.
 SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
   DAGCombinerInfo &DCI, const SDLoc &SL,
   unsigned Opc, SDValue LHS,
   uint32_t ValLo, uint32_t ValHi) const {
   SelectionDAG &DAG = DCI.DAG;
   SDValue Lo, Hi;
   std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
 
   SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
   SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
 
   SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
   SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
 
   // Re-visit the ands. It's possible we eliminated one of them and it could
   // simplify the vector.
   DCI.AddToWorklist(Lo.getNode());
   DCI.AddToWorklist(Hi.getNode());
 
   SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
   return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
 }
 
 SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
   ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
   if (!RHS)
     return SDValue();
 
   SDValue LHS = N->getOperand(0);
   unsigned RHSVal = RHS->getZExtValue();
   if (!RHSVal)
     return LHS;
 
   SDLoc SL(N);
   SelectionDAG &DAG = DCI.DAG;
 
   switch (LHS->getOpcode()) {
   default:
     break;
   case ISD::ZERO_EXTEND:
   case ISD::SIGN_EXTEND:
   case ISD::ANY_EXTEND: {
     SDValue X = LHS->getOperand(0);
 
     if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
         isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) {
       // Prefer build_vector as the canonical form if packed types are legal.
       // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
       SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
        { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) });
       return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
     }
 
     // shl (ext x) => zext (shl x), if shift does not overflow int
     if (VT != MVT::i64)
       break;
     KnownBits Known = DAG.computeKnownBits(X);
     unsigned LZ = Known.countMinLeadingZeros();
     if (LZ < RHSVal)
       break;
     EVT XVT = X.getValueType();
     SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
     return DAG.getZExtOrTrunc(Shl, SL, VT);
   }
   }
 
   if (VT != MVT::i64)
     return SDValue();
 
   // i64 (shl x, C) -> (build_pair 0, (shl x, C -32))
 
   // On some subtargets, 64-bit shift is a quarter rate instruction. In the
   // common case, splitting this into a move and a 32-bit shift is faster and
   // the same code size.
   if (RHSVal < 32)
     return SDValue();
 
   SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
 
   SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
   SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
 
   const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
 
   SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
   return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
 }
 
 SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   if (N->getValueType(0) != MVT::i64)
     return SDValue();
 
   const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
   if (!RHS)
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   SDLoc SL(N);
   unsigned RHSVal = RHS->getZExtValue();
 
   // (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31)
   if (RHSVal == 32) {
     SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
     SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
                                    DAG.getConstant(31, SL, MVT::i32));
 
     SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
     return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
   }
 
   // (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31)
   if (RHSVal == 63) {
     SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
     SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
                                    DAG.getConstant(31, SL, MVT::i32));
     SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
     return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
   }
 
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
   if (!RHS)
     return SDValue();
 
   EVT VT = N->getValueType(0);
   SDValue LHS = N->getOperand(0);
   unsigned ShiftAmt = RHS->getZExtValue();
   SelectionDAG &DAG = DCI.DAG;
   SDLoc SL(N);
 
   // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
   // this improves the ability to match BFE patterns in isel.
   if (LHS.getOpcode() == ISD::AND) {
     if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
       if (Mask->getAPIntValue().isShiftedMask() &&
           Mask->getAPIntValue().countTrailingZeros() == ShiftAmt) {
         return DAG.getNode(
             ISD::AND, SL, VT,
             DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
             DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
       }
     }
   }
 
   if (VT != MVT::i64)
     return SDValue();
 
   if (ShiftAmt < 32)
     return SDValue();
 
   // srl i64:x, C for C >= 32
   // =>
   //   build_pair (srl hi_32(x), C - 32), 0
   SDValue One = DAG.getConstant(1, SL, MVT::i32);
   SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
 
   SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, LHS);
   SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecOp, One);
 
   SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
   SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
 
   SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
 
   return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
 }
 
 SDValue AMDGPUTargetLowering::performTruncateCombine(
   SDNode *N, DAGCombinerInfo &DCI) const {
   SDLoc SL(N);
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
   SDValue Src = N->getOperand(0);
 
   // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
   if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
     SDValue Vec = Src.getOperand(0);
     if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
       SDValue Elt0 = Vec.getOperand(0);
       EVT EltVT = Elt0.getValueType();
       if (VT.getSizeInBits() <= EltVT.getSizeInBits()) {
         if (EltVT.isFloatingPoint()) {
           Elt0 = DAG.getNode(ISD::BITCAST, SL,
                              EltVT.changeTypeToInteger(), Elt0);
         }
 
         return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
       }
     }
   }
 
   // Equivalent of above for accessing the high element of a vector as an
   // integer operation.
   // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
   if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
     if (auto K = isConstOrConstSplat(Src.getOperand(1))) {
       if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
         SDValue BV = stripBitcast(Src.getOperand(0));
         if (BV.getOpcode() == ISD::BUILD_VECTOR &&
             BV.getValueType().getVectorNumElements() == 2) {
           SDValue SrcElt = BV.getOperand(1);
           EVT SrcEltVT = SrcElt.getValueType();
           if (SrcEltVT.isFloatingPoint()) {
             SrcElt = DAG.getNode(ISD::BITCAST, SL,
                                  SrcEltVT.changeTypeToInteger(), SrcElt);
           }
 
           return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
         }
       }
     }
   }
 
   // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
   //
   // i16 (trunc (srl i64:x, K)), K <= 16 ->
   //     i16 (trunc (srl (i32 (trunc x), K)))
   if (VT.getScalarSizeInBits() < 32) {
     EVT SrcVT = Src.getValueType();
     if (SrcVT.getScalarSizeInBits() > 32 &&
         (Src.getOpcode() == ISD::SRL ||
          Src.getOpcode() == ISD::SRA ||
          Src.getOpcode() == ISD::SHL)) {
       SDValue Amt = Src.getOperand(1);
       KnownBits Known = DAG.computeKnownBits(Amt);
       unsigned Size = VT.getScalarSizeInBits();
       if ((Known.isConstant() && Known.getConstant().ule(Size)) ||
           (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size))) {
         EVT MidVT = VT.isVector() ?
           EVT::getVectorVT(*DAG.getContext(), MVT::i32,
                            VT.getVectorNumElements()) : MVT::i32;
 
         EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout());
         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
                                     Src.getOperand(0));
         DCI.AddToWorklist(Trunc.getNode());
 
         if (Amt.getValueType() != NewShiftVT) {
           Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT);
           DCI.AddToWorklist(Amt.getNode());
         }
 
         SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
                                           Trunc, Amt);
         return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
       }
     }
   }
 
   return SDValue();
 }
 
 // We need to specifically handle i64 mul here to avoid unnecessary conversion
 // instructions. If we only match on the legalized i64 mul expansion,
 // SimplifyDemandedBits will be unable to remove them because there will be
 // multiple uses due to the separate mul + mulh[su].
 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
                         SDValue N0, SDValue N1, unsigned Size, bool Signed) {
   if (Size <= 32) {
     unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
     return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
   }
 
   // Because we want to eliminate extension instructions before the
   // operation, we need to create a single user here (i.e. not the separate
   // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it.
 
   unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24;
 
   SDValue Mul = DAG.getNode(MulOpc, SL,
                             DAG.getVTList(MVT::i32, MVT::i32), N0, N1);
 
   return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64,
                      Mul.getValue(0), Mul.getValue(1));
 }
 
 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
   unsigned Size = VT.getSizeInBits();
   if (VT.isVector() || Size > 64)
     return SDValue();
 
   // There are i16 integer mul/mad.
   if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
   // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
   // in the source into any_extends if the result of the mul is truncated. Since
   // we can assume the high bits are whatever we want, use the underlying value
   // to avoid the unknown high bits from interfering.
   if (N0.getOpcode() == ISD::ANY_EXTEND)
     N0 = N0.getOperand(0);
 
   if (N1.getOpcode() == ISD::ANY_EXTEND)
     N1 = N1.getOperand(0);
 
   SDValue Mul;
 
   if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
     N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
     N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
     Mul = getMul24(DAG, DL, N0, N1, Size, false);
   } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
     N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
     N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
     Mul = getMul24(DAG, DL, N0, N1, Size, true);
   } else {
     return SDValue();
   }
 
   // We need to use sext even for MUL_U24, because MUL_U24 is used
   // for signed multiply of 8 and 16-bit types.
   return DAG.getSExtOrTrunc(Mul, DL, VT);
 }
 
 SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
                                                   DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
   if (!Subtarget->hasMulI24() || VT.isVector())
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
   if (!isI24(N0, DAG) || !isI24(N1, DAG))
     return SDValue();
 
   N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
   N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
 
   SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
   DCI.AddToWorklist(Mulhi.getNode());
   return DAG.getSExtOrTrunc(Mulhi, DL, VT);
 }
 
 SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
                                                   DAGCombinerInfo &DCI) const {
   EVT VT = N->getValueType(0);
 
   if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
   if (!isU24(N0, DAG) || !isU24(N1, DAG))
     return SDValue();
 
   N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
   N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
 
   SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
   DCI.AddToWorklist(Mulhi.getNode());
   return DAG.getZExtOrTrunc(Mulhi, DL, VT);
 }
 
 SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
   SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
 
   // Simplify demanded bits before splitting into multiple users.
   if (SDValue V = simplifyI24(N, DCI))
     return V;
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
 
   bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24);
 
   unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
   unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
 
   SDLoc SL(N);
 
   SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
   SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
   return DAG.getMergeValues({ MulLo, MulHi }, SL);
 }
 
 static bool isNegativeOne(SDValue Val) {
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
     return C->isAllOnesValue();
   return false;
 }
 
 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
                                           SDValue Op,
                                           const SDLoc &DL,
                                           unsigned Opc) const {
   EVT VT = Op.getValueType();
   EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
   if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() &&
                               LegalVT != MVT::i16))
     return SDValue();
 
   if (VT != MVT::i32)
     Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);
 
   SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
   if (VT != MVT::i32)
     FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);
 
   return FFBX;
 }
 
 // The native instructions return -1 on 0 input. Optimize out a select that
 // produces -1 on 0.
 //
 // TODO: If zero is not undef, we could also do this if the output is compared
 // against the bitwidth.
 //
 // TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
                                                  SDValue LHS, SDValue RHS,
                                                  DAGCombinerInfo &DCI) const {
   ConstantSDNode *CmpRhs = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
   if (!CmpRhs || !CmpRhs->isNullValue())
     return SDValue();
 
   SelectionDAG &DAG = DCI.DAG;
   ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
   SDValue CmpLHS = Cond.getOperand(0);
 
   unsigned Opc = isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 :
                                            AMDGPUISD::FFBH_U32;
 
   // select (setcc x, 0, eq), -1, (ctlz_zero_undef x) -> ffbh_u32 x
   // select (setcc x, 0, eq), -1, (cttz_zero_undef x) -> ffbl_u32 x
   if (CCOpcode == ISD::SETEQ &&
       (isCtlzOpc(RHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
       RHS.getOperand(0) == CmpLHS &&
       isNegativeOne(LHS)) {
     return getFFBX_U32(DAG, CmpLHS, SL, Opc);
   }
 
   // select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x
   // select (setcc x, 0, ne), (cttz_zero_undef x), -1 -> ffbl_u32 x
   if (CCOpcode == ISD::SETNE &&
       (isCtlzOpc(LHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
       LHS.getOperand(0) == CmpLHS &&
       isNegativeOne(RHS)) {
     return getFFBX_U32(DAG, CmpLHS, SL, Opc);
   }
 
   return SDValue();
 }
 
 static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
                                          unsigned Op,
                                          const SDLoc &SL,
                                          SDValue Cond,
                                          SDValue N1,
                                          SDValue N2) {
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N1.getValueType();
 
   SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
                                   N1.getOperand(0), N2.getOperand(0));
   DCI.AddToWorklist(NewSelect.getNode());
   return DAG.getNode(Op, SL, VT, NewSelect);
 }
 
 // Pull a free FP operation out of a select so it may fold into uses.
 //
 // select c, (fneg x), (fneg y) -> fneg (select c, x, y)
 // select c, (fneg x), k -> fneg (select c, x, (fneg k))
 //
 // select c, (fabs x), (fabs y) -> fabs (select c, x, y)
 // select c, (fabs x), +k -> fabs (select c, x, k)
 static SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
                                     SDValue N) {
   SelectionDAG &DAG = DCI.DAG;
   SDValue Cond = N.getOperand(0);
   SDValue LHS = N.getOperand(1);
   SDValue RHS = N.getOperand(2);
 
   EVT VT = N.getValueType();
   if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
       (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
     return distributeOpThroughSelect(DCI, LHS.getOpcode(),
                                      SDLoc(N), Cond, LHS, RHS);
   }
 
   bool Inv = false;
   if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
     std::swap(LHS, RHS);
     Inv = true;
   }
 
   // TODO: Support vector constants.
   ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS);
   if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS) {
     SDLoc SL(N);
     // If one side is an fneg/fabs and the other is a constant, we can push the
     // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
     SDValue NewLHS = LHS.getOperand(0);
     SDValue NewRHS = RHS;
 
     // Careful: if the neg can be folded up, don't try to pull it back down.
     bool ShouldFoldNeg = true;
 
     if (NewLHS.hasOneUse()) {
       unsigned Opc = NewLHS.getOpcode();
       if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(Opc))
         ShouldFoldNeg = false;
       if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
         ShouldFoldNeg = false;
     }
 
     if (ShouldFoldNeg) {
       if (LHS.getOpcode() == ISD::FNEG)
         NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
       else if (CRHS->isNegative())
         return SDValue();
 
       if (Inv)
         std::swap(NewLHS, NewRHS);
 
       SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
                                       Cond, NewLHS, NewRHS);
       DCI.AddToWorklist(NewSelect.getNode());
       return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
     }
   }
 
   return SDValue();
 }
 
 
 SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
                                                    DAGCombinerInfo &DCI) const {
   if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
     return Folded;
 
   SDValue Cond = N->getOperand(0);
   if (Cond.getOpcode() != ISD::SETCC)
     return SDValue();
 
   EVT VT = N->getValueType(0);
   SDValue LHS = Cond.getOperand(0);
   SDValue RHS = Cond.getOperand(1);
   SDValue CC = Cond.getOperand(2);
 
   SDValue True = N->getOperand(1);
   SDValue False = N->getOperand(2);
 
   if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
     SelectionDAG &DAG = DCI.DAG;
     if (DAG.isConstantValueOfAnyType(True) &&
         !DAG.isConstantValueOfAnyType(False)) {
       // Swap cmp + select pair to move constant to false input.
       // This will allow using VOPC cndmasks more often.
       // select (setcc x, y), k, x -> select (setccinv x, y), x, k
 
       SDLoc SL(N);
       ISD::CondCode NewCC =
           getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), LHS.getValueType());
 
       SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
       return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
     }
 
     if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
       SDValue MinMax
         = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
       // Revisit this node so we can catch min3/max3/med3 patterns.
       //DCI.AddToWorklist(MinMax.getNode());
       return MinMax;
     }
   }
 
   // There's no reason to not do this if the condition has other uses.
   return performCtlz_CttzCombine(SDLoc(N), Cond, True, False, DCI);
 }
 
 static bool isInv2Pi(const APFloat &APF) {
   static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
   static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
   static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
 
   return APF.bitwiseIsEqual(KF16) ||
          APF.bitwiseIsEqual(KF32) ||
          APF.bitwiseIsEqual(KF64);
 }
 
 // 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
 // additional cost to negate them.
 bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
   if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N)) {
     if (C->isZero() && !C->isNegative())
       return true;
 
     if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(C->getValueAPF()))
       return true;
   }
 
   return false;
 }
 
 static unsigned inverseMinMax(unsigned Opc) {
   switch (Opc) {
   case ISD::FMAXNUM:
     return ISD::FMINNUM;
   case ISD::FMINNUM:
     return ISD::FMAXNUM;
   case ISD::FMAXNUM_IEEE:
     return ISD::FMINNUM_IEEE;
   case ISD::FMINNUM_IEEE:
     return ISD::FMAXNUM_IEEE;
   case AMDGPUISD::FMAX_LEGACY:
     return AMDGPUISD::FMIN_LEGACY;
   case AMDGPUISD::FMIN_LEGACY:
     return  AMDGPUISD::FMAX_LEGACY;
   default:
     llvm_unreachable("invalid min/max opcode");
   }
 }
 
 SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
   unsigned Opc = N0.getOpcode();
 
   // If the input has multiple uses and we can either fold the negate down, or
   // the other uses cannot, give up. This both prevents unprofitable
   // transformations and infinite loops: we won't repeatedly try to fold around
   // a negate that has no 'good' form.
   if (N0.hasOneUse()) {
     // This may be able to fold into the source, but at a code size cost. Don't
     // fold if the fold into the user is free.
     if (allUsesHaveSourceMods(N, 0))
       return SDValue();
   } else {
     if (fnegFoldsIntoOp(Opc) &&
         (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N0.getNode())))
       return SDValue();
   }
 
   SDLoc SL(N);
   switch (Opc) {
   case ISD::FADD: {
     if (!mayIgnoreSignedZero(N0))
       return SDValue();
 
     // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
     SDValue LHS = N0.getOperand(0);
     SDValue RHS = N0.getOperand(1);
 
     if (LHS.getOpcode() != ISD::FNEG)
       LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
     else
       LHS = LHS.getOperand(0);
 
     if (RHS.getOpcode() != ISD::FNEG)
       RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
     else
       RHS = RHS.getOperand(0);
 
     SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
     if (Res.getOpcode() != ISD::FADD)
       return SDValue(); // Op got folded away.
     if (!N0.hasOneUse())
       DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
     return Res;
   }
   case ISD::FMUL:
   case AMDGPUISD::FMUL_LEGACY: {
     // (fneg (fmul x, y)) -> (fmul x, (fneg y))
     // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
     SDValue LHS = N0.getOperand(0);
     SDValue RHS = N0.getOperand(1);
 
     if (LHS.getOpcode() == ISD::FNEG)
       LHS = LHS.getOperand(0);
     else if (RHS.getOpcode() == ISD::FNEG)
       RHS = RHS.getOperand(0);
     else
       RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
 
     SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
     if (Res.getOpcode() != Opc)
       return SDValue(); // Op got folded away.
     if (!N0.hasOneUse())
       DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
     return Res;
   }
   case ISD::FMA:
   case ISD::FMAD: {
     if (!mayIgnoreSignedZero(N0))
       return SDValue();
 
     // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
     SDValue LHS = N0.getOperand(0);
     SDValue MHS = N0.getOperand(1);
     SDValue RHS = N0.getOperand(2);
 
     if (LHS.getOpcode() == ISD::FNEG)
       LHS = LHS.getOperand(0);
     else if (MHS.getOpcode() == ISD::FNEG)
       MHS = MHS.getOperand(0);
     else
       MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
 
     if (RHS.getOpcode() != ISD::FNEG)
       RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
     else
       RHS = RHS.getOperand(0);
 
     SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
     if (Res.getOpcode() != Opc)
       return SDValue(); // Op got folded away.
     if (!N0.hasOneUse())
       DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
     return Res;
   }
   case ISD::FMAXNUM:
   case ISD::FMINNUM:
   case ISD::FMAXNUM_IEEE:
   case ISD::FMINNUM_IEEE:
   case AMDGPUISD::FMAX_LEGACY:
   case AMDGPUISD::FMIN_LEGACY: {
     // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
     // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
     // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
     // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
 
     SDValue LHS = N0.getOperand(0);
     SDValue RHS = N0.getOperand(1);
 
     // 0 doesn't have a negated inline immediate.
     // TODO: This constant check should be generalized to other operations.
     if (isConstantCostlierToNegate(RHS))
       return SDValue();
 
     SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
     SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
     unsigned Opposite = inverseMinMax(Opc);
 
     SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
     if (Res.getOpcode() != Opposite)
       return SDValue(); // Op got folded away.
     if (!N0.hasOneUse())
       DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
     return Res;
   }
   case AMDGPUISD::FMED3: {
     SDValue Ops[3];
     for (unsigned I = 0; I < 3; ++I)
       Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
 
     SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
     if (Res.getOpcode() != AMDGPUISD::FMED3)
       return SDValue(); // Op got folded away.
     if (!N0.hasOneUse())
       DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
     return Res;
   }
   case ISD::FP_EXTEND:
   case ISD::FTRUNC:
   case ISD::FRINT:
   case ISD::FNEARBYINT: // XXX - Should fround be handled?
   case ISD::FSIN:
   case ISD::FCANONICALIZE:
   case AMDGPUISD::RCP:
   case AMDGPUISD::RCP_LEGACY:
   case AMDGPUISD::RCP_IFLAG:
   case AMDGPUISD::SIN_HW: {
     SDValue CvtSrc = N0.getOperand(0);
     if (CvtSrc.getOpcode() == ISD::FNEG) {
       // (fneg (fp_extend (fneg x))) -> (fp_extend x)
       // (fneg (rcp (fneg x))) -> (rcp x)
       return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
     }
 
     if (!N0.hasOneUse())
       return SDValue();
 
     // (fneg (fp_extend x)) -> (fp_extend (fneg x))
     // (fneg (rcp x)) -> (rcp (fneg x))
     SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
     return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
   }
   case ISD::FP_ROUND: {
     SDValue CvtSrc = N0.getOperand(0);
 
     if (CvtSrc.getOpcode() == ISD::FNEG) {
       // (fneg (fp_round (fneg x))) -> (fp_round x)
       return DAG.getNode(ISD::FP_ROUND, SL, VT,
                          CvtSrc.getOperand(0), N0.getOperand(1));
     }
 
     if (!N0.hasOneUse())
       return SDValue();
 
     // (fneg (fp_round x)) -> (fp_round (fneg x))
     SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
     return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
   }
   case ISD::FP16_TO_FP: {
     // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
     // f16, but legalization of f16 fneg ends up pulling it out of the source.
     // Put the fneg back as a legal source operation that can be matched later.
     SDLoc SL(N);
 
     SDValue Src = N0.getOperand(0);
     EVT SrcVT = Src.getValueType();
 
     // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
     SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
                                   DAG.getConstant(0x8000, SL, SrcVT));
     return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
   }
   default:
     return SDValue();
   }
 }
 
 SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
 
   if (!N0.hasOneUse())
     return SDValue();
 
   switch (N0.getOpcode()) {
   case ISD::FP16_TO_FP: {
     assert(!Subtarget->has16BitInsts() && "should only see if f16 is illegal");
     SDLoc SL(N);
     SDValue Src = N0.getOperand(0);
     EVT SrcVT = Src.getValueType();
 
     // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
     SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
                                   DAG.getConstant(0x7fff, SL, SrcVT));
     return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
   }
   default:
     return SDValue();
   }
 }
 
 SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
   if (!CFP)
     return SDValue();
 
   // XXX - Should this flush denormals?
   const APFloat &Val = CFP->getValueAPF();
   APFloat One(Val.getSemantics(), "1.0");
   return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
 }
 
 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
   switch(N->getOpcode()) {
   default:
     break;
   case ISD::BITCAST: {
     EVT DestVT = N->getValueType(0);
 
     // Push casts through vector builds. This helps avoid emitting a large
     // number of copies when materializing floating point vector constants.
     //
     // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
     //   vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
     if (DestVT.isVector()) {
       SDValue Src = N->getOperand(0);
       if (Src.getOpcode() == ISD::BUILD_VECTOR) {
         EVT SrcVT = Src.getValueType();
         unsigned NElts = DestVT.getVectorNumElements();
 
         if (SrcVT.getVectorNumElements() == NElts) {
           EVT DestEltVT = DestVT.getVectorElementType();
 
           SmallVector<SDValue, 8> CastedElts;
           SDLoc SL(N);
           for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
             SDValue Elt = Src.getOperand(I);
             CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
           }
 
           return DAG.getBuildVector(DestVT, SL, CastedElts);
         }
       }
     }
 
     if (DestVT.getSizeInBits() != 64 && !DestVT.isVector())
       break;
 
     // Fold bitcasts of constants.
     //
     // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
     // TODO: Generalize and move to DAGCombiner
     SDValue Src = N->getOperand(0);
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src)) {
       if (Src.getValueType() == MVT::i64) {
         SDLoc SL(N);
         uint64_t CVal = C->getZExtValue();
         SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
                                  DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
                                  DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
         return DAG.getNode(ISD::BITCAST, SL, DestVT, BV);
       }
     }
 
     if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Src)) {
       const APInt &Val = C->getValueAPF().bitcastToAPInt();
       SDLoc SL(N);
       uint64_t CVal = Val.getZExtValue();
       SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
                                 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
                                 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
 
       return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
     }
 
     break;
   }
   case ISD::SHL: {
     if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
 
     return performShlCombine(N, DCI);
   }
   case ISD::SRL: {
     if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
 
     return performSrlCombine(N, DCI);
   }
   case ISD::SRA: {
     if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
       break;
 
     return performSraCombine(N, DCI);
   }
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
   case ISD::MUL:
     return performMulCombine(N, DCI);
   case ISD::MULHS:
     return performMulhsCombine(N, DCI);
   case ISD::MULHU:
     return performMulhuCombine(N, DCI);
   case AMDGPUISD::MUL_I24:
   case AMDGPUISD::MUL_U24:
   case AMDGPUISD::MULHI_I24:
   case AMDGPUISD::MULHI_U24: {
     if (SDValue V = simplifyI24(N, DCI))
       return V;
     return SDValue();
   }
   case AMDGPUISD::MUL_LOHI_I24:
   case AMDGPUISD::MUL_LOHI_U24:
     return performMulLoHi24Combine(N, DCI);
   case ISD::SELECT:
     return performSelectCombine(N, DCI);
   case ISD::FNEG:
     return performFNegCombine(N, DCI);
   case ISD::FABS:
     return performFAbsCombine(N, DCI);
   case AMDGPUISD::BFE_I32:
   case AMDGPUISD::BFE_U32: {
     assert(!N->getValueType(0).isVector() &&
            "Vector handling of BFE not implemented");
     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
     if (!Width)
       break;
 
     uint32_t WidthVal = Width->getZExtValue() & 0x1f;
     if (WidthVal == 0)
       return DAG.getConstant(0, DL, MVT::i32);
 
     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
     if (!Offset)
       break;
 
     SDValue BitsFrom = N->getOperand(0);
     uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
 
     bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
 
     if (OffsetVal == 0) {
       // This is already sign / zero extended, so try to fold away extra BFEs.
       unsigned SignBits =  Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
 
       unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
       if (OpSignBits >= SignBits)
         return BitsFrom;
 
       EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
       if (Signed) {
         // This is a sign_extend_inreg. Replace it to take advantage of existing
         // DAG Combines. If not eliminated, we will match back to BFE during
         // selection.
 
         // TODO: The sext_inreg of extended types ends, although we can could
         // handle them in a single BFE.
         return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
                            DAG.getValueType(SmallVT));
       }
 
       return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
     }
 
     if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
       if (Signed) {
         return constantFoldBFE<int32_t>(DAG,
                                         CVal->getSExtValue(),
                                         OffsetVal,
                                         WidthVal,
                                         DL);
       }
 
       return constantFoldBFE<uint32_t>(DAG,
                                        CVal->getZExtValue(),
                                        OffsetVal,
                                        WidthVal,
                                        DL);
     }
 
     if ((OffsetVal + WidthVal) >= 32 &&
         !(Subtarget->hasSDWA() && OffsetVal == 16 && WidthVal == 16)) {
       SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
       return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
                          BitsFrom, ShiftVal);
     }
 
     if (BitsFrom.hasOneUse()) {
       APInt Demanded = APInt::getBitsSet(32,
                                          OffsetVal,
                                          OffsetVal + WidthVal);
 
       KnownBits Known;
       TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                                             !DCI.isBeforeLegalizeOps());
       const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       if (TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) ||
           TLI.SimplifyDemandedBits(BitsFrom, Demanded, Known, TLO)) {
         DCI.CommitTargetLoweringOpt(TLO);
       }
     }
 
     break;
   }
   case ISD::LOAD:
     return performLoadCombine(N, DCI);
   case ISD::STORE:
     return performStoreCombine(N, DCI);
   case AMDGPUISD::RCP:
   case AMDGPUISD::RCP_IFLAG:
     return performRcpCombine(N, DCI);
   case ISD::AssertZext:
   case ISD::AssertSext:
     return performAssertSZExtCombine(N, DCI);
   case ISD::INTRINSIC_WO_CHAIN:
     return performIntrinsicWOChainCombine(N, DCI);
   }
   return SDValue();
 }
 
 //===----------------------------------------------------------------------===//
 // Helper functions
 //===----------------------------------------------------------------------===//
 
 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
                                                    const TargetRegisterClass *RC,
                                                    Register Reg, EVT VT,
                                                    const SDLoc &SL,
                                                    bool RawReg) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   Register VReg;
 
   if (!MRI.isLiveIn(Reg)) {
     VReg = MRI.createVirtualRegister(RC);
     MRI.addLiveIn(Reg, VReg);
   } else {
     VReg = MRI.getLiveInVirtReg(Reg);
   }
 
   if (RawReg)
     return DAG.getRegister(VReg, VT);
 
   return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
 }
 
 // This may be called multiple times, and nothing prevents creating multiple
 // objects at the same offset. See if we already defined this object.
 static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
                                        int64_t Offset) {
   for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
     if (MFI.getObjectOffset(I) == Offset) {
       assert(MFI.getObjectSize(I) == Size);
       return I;
     }
   }
 
   return MFI.CreateFixedObject(Size, Offset, true);
 }
 
 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
                                                   EVT VT,
                                                   const SDLoc &SL,
                                                   int64_t Offset) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset);
 
   auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
   SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
 
   return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, 4,
                      MachineMemOperand::MODereferenceable |
                      MachineMemOperand::MOInvariant);
 }
 
 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
                                                    const SDLoc &SL,
                                                    SDValue Chain,
                                                    SDValue ArgVal,
                                                    int64_t Offset) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
 
   SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
   SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, 4,
                                MachineMemOperand::MODereferenceable);
   return Store;
 }
 
 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
                                              const TargetRegisterClass *RC,
                                              EVT VT, const SDLoc &SL,
                                              const ArgDescriptor &Arg) const {
   assert(Arg && "Attempting to load missing argument");
 
   SDValue V = Arg.isRegister() ?
     CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
     loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
 
   if (!Arg.isMasked())
     return V;
 
   unsigned Mask = Arg.getMask();
   unsigned Shift = countTrailingZeros<unsigned>(Mask);
   V = DAG.getNode(ISD::SRL, SL, VT, V,
                   DAG.getShiftAmountConstant(Shift, VT, SL));
   return DAG.getNode(ISD::AND, SL, VT, V,
                      DAG.getConstant(Mask >> Shift, SL, VT));
 }
 
 uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
     const MachineFunction &MF, const ImplicitParameter Param) const {
   const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
   const AMDGPUSubtarget &ST =
       AMDGPUSubtarget::get(getTargetMachine(), MF.getFunction());
   unsigned ExplicitArgOffset = ST.getExplicitKernelArgOffset(MF.getFunction());
   const Align Alignment = ST.getAlignmentForImplicitArgPtr();
   uint64_t ArgOffset = alignTo(MFI->getExplicitKernArgSize(), Alignment) +
                        ExplicitArgOffset;
   switch (Param) {
   case GRID_DIM:
     return ArgOffset;
   case GRID_OFFSET:
     return ArgOffset + 4;
   }
   llvm_unreachable("unexpected implicit parameter type");
 }
 
 #define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
 
 const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   switch ((AMDGPUISD::NodeType)Opcode) {
   case AMDGPUISD::FIRST_NUMBER: break;
   // AMDIL DAG nodes
   NODE_NAME_CASE(UMUL);
   NODE_NAME_CASE(BRANCH_COND);
 
   // AMDGPU DAG nodes
   NODE_NAME_CASE(IF)
   NODE_NAME_CASE(ELSE)
   NODE_NAME_CASE(LOOP)
   NODE_NAME_CASE(CALL)
   NODE_NAME_CASE(TC_RETURN)
   NODE_NAME_CASE(TRAP)
   NODE_NAME_CASE(RET_FLAG)
   NODE_NAME_CASE(RETURN_TO_EPILOG)
   NODE_NAME_CASE(ENDPGM)
   NODE_NAME_CASE(DWORDADDR)
   NODE_NAME_CASE(FRACT)
   NODE_NAME_CASE(SETCC)
   NODE_NAME_CASE(SETREG)
   NODE_NAME_CASE(DENORM_MODE)
   NODE_NAME_CASE(FMA_W_CHAIN)
   NODE_NAME_CASE(FMUL_W_CHAIN)
   NODE_NAME_CASE(CLAMP)
   NODE_NAME_CASE(COS_HW)
   NODE_NAME_CASE(SIN_HW)
   NODE_NAME_CASE(FMAX_LEGACY)
   NODE_NAME_CASE(FMIN_LEGACY)
   NODE_NAME_CASE(FMAX3)
   NODE_NAME_CASE(SMAX3)
   NODE_NAME_CASE(UMAX3)
   NODE_NAME_CASE(FMIN3)
   NODE_NAME_CASE(SMIN3)
   NODE_NAME_CASE(UMIN3)
   NODE_NAME_CASE(FMED3)
   NODE_NAME_CASE(SMED3)
   NODE_NAME_CASE(UMED3)
   NODE_NAME_CASE(FDOT2)
   NODE_NAME_CASE(URECIP)
   NODE_NAME_CASE(DIV_SCALE)
   NODE_NAME_CASE(DIV_FMAS)
   NODE_NAME_CASE(DIV_FIXUP)
   NODE_NAME_CASE(FMAD_FTZ)
   NODE_NAME_CASE(TRIG_PREOP)
   NODE_NAME_CASE(RCP)
   NODE_NAME_CASE(RSQ)
   NODE_NAME_CASE(RCP_LEGACY)
   NODE_NAME_CASE(RCP_IFLAG)
   NODE_NAME_CASE(FMUL_LEGACY)
   NODE_NAME_CASE(RSQ_CLAMP)
   NODE_NAME_CASE(LDEXP)
   NODE_NAME_CASE(FP_CLASS)
   NODE_NAME_CASE(DOT4)
   NODE_NAME_CASE(CARRY)
   NODE_NAME_CASE(BORROW)
   NODE_NAME_CASE(BFE_U32)
   NODE_NAME_CASE(BFE_I32)
   NODE_NAME_CASE(BFI)
   NODE_NAME_CASE(BFM)
   NODE_NAME_CASE(FFBH_U32)
   NODE_NAME_CASE(FFBH_I32)
   NODE_NAME_CASE(FFBL_B32)
   NODE_NAME_CASE(MUL_U24)
   NODE_NAME_CASE(MUL_I24)
   NODE_NAME_CASE(MULHI_U24)
   NODE_NAME_CASE(MULHI_I24)
   NODE_NAME_CASE(MUL_LOHI_U24)
   NODE_NAME_CASE(MUL_LOHI_I24)
   NODE_NAME_CASE(MAD_U24)
   NODE_NAME_CASE(MAD_I24)
   NODE_NAME_CASE(MAD_I64_I32)
   NODE_NAME_CASE(MAD_U64_U32)
   NODE_NAME_CASE(PERM)
   NODE_NAME_CASE(TEXTURE_FETCH)
   NODE_NAME_CASE(R600_EXPORT)
   NODE_NAME_CASE(CONST_ADDRESS)
   NODE_NAME_CASE(REGISTER_LOAD)
   NODE_NAME_CASE(REGISTER_STORE)
   NODE_NAME_CASE(SAMPLE)
   NODE_NAME_CASE(SAMPLEB)
   NODE_NAME_CASE(SAMPLED)
   NODE_NAME_CASE(SAMPLEL)
   NODE_NAME_CASE(CVT_F32_UBYTE0)
   NODE_NAME_CASE(CVT_F32_UBYTE1)
   NODE_NAME_CASE(CVT_F32_UBYTE2)
   NODE_NAME_CASE(CVT_F32_UBYTE3)
   NODE_NAME_CASE(CVT_PKRTZ_F16_F32)
   NODE_NAME_CASE(CVT_PKNORM_I16_F32)
   NODE_NAME_CASE(CVT_PKNORM_U16_F32)
   NODE_NAME_CASE(CVT_PK_I16_I32)
   NODE_NAME_CASE(CVT_PK_U16_U32)
   NODE_NAME_CASE(FP_TO_FP16)
   NODE_NAME_CASE(FP16_ZEXT)
   NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)
   NODE_NAME_CASE(CONST_DATA_PTR)
   NODE_NAME_CASE(PC_ADD_REL_OFFSET)
   NODE_NAME_CASE(LDS)
   NODE_NAME_CASE(DUMMY_CHAIN)
   case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
   NODE_NAME_CASE(LOAD_D16_HI)
   NODE_NAME_CASE(LOAD_D16_LO)
   NODE_NAME_CASE(LOAD_D16_HI_I8)
   NODE_NAME_CASE(LOAD_D16_HI_U8)
   NODE_NAME_CASE(LOAD_D16_LO_I8)
   NODE_NAME_CASE(LOAD_D16_LO_U8)
   NODE_NAME_CASE(STORE_MSKOR)
   NODE_NAME_CASE(LOAD_CONSTANT)
   NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
   NODE_NAME_CASE(TBUFFER_STORE_FORMAT_D16)
   NODE_NAME_CASE(TBUFFER_LOAD_FORMAT)
   NODE_NAME_CASE(TBUFFER_LOAD_FORMAT_D16)
   NODE_NAME_CASE(DS_ORDERED_COUNT)
   NODE_NAME_CASE(ATOMIC_CMP_SWAP)
   NODE_NAME_CASE(ATOMIC_INC)
   NODE_NAME_CASE(ATOMIC_DEC)
   NODE_NAME_CASE(ATOMIC_LOAD_FMIN)
   NODE_NAME_CASE(ATOMIC_LOAD_FMAX)
   NODE_NAME_CASE(BUFFER_LOAD)
   NODE_NAME_CASE(BUFFER_LOAD_UBYTE)
   NODE_NAME_CASE(BUFFER_LOAD_USHORT)
   NODE_NAME_CASE(BUFFER_LOAD_BYTE)
   NODE_NAME_CASE(BUFFER_LOAD_SHORT)
   NODE_NAME_CASE(BUFFER_LOAD_FORMAT)
   NODE_NAME_CASE(BUFFER_LOAD_FORMAT_D16)
   NODE_NAME_CASE(SBUFFER_LOAD)
   NODE_NAME_CASE(BUFFER_STORE)
   NODE_NAME_CASE(BUFFER_STORE_BYTE)
   NODE_NAME_CASE(BUFFER_STORE_SHORT)
   NODE_NAME_CASE(BUFFER_STORE_FORMAT)
   NODE_NAME_CASE(BUFFER_STORE_FORMAT_D16)
   NODE_NAME_CASE(BUFFER_ATOMIC_SWAP)
   NODE_NAME_CASE(BUFFER_ATOMIC_ADD)
   NODE_NAME_CASE(BUFFER_ATOMIC_SUB)
   NODE_NAME_CASE(BUFFER_ATOMIC_SMIN)
   NODE_NAME_CASE(BUFFER_ATOMIC_UMIN)
   NODE_NAME_CASE(BUFFER_ATOMIC_SMAX)
   NODE_NAME_CASE(BUFFER_ATOMIC_UMAX)
   NODE_NAME_CASE(BUFFER_ATOMIC_AND)
   NODE_NAME_CASE(BUFFER_ATOMIC_OR)
   NODE_NAME_CASE(BUFFER_ATOMIC_XOR)
   NODE_NAME_CASE(BUFFER_ATOMIC_INC)
   NODE_NAME_CASE(BUFFER_ATOMIC_DEC)
   NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP)
   NODE_NAME_CASE(BUFFER_ATOMIC_FADD)
   NODE_NAME_CASE(BUFFER_ATOMIC_PK_FADD)
   NODE_NAME_CASE(ATOMIC_PK_FADD)
 
   case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
   }
   return nullptr;
 }
 
 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
                                               SelectionDAG &DAG, int Enabled,
                                               int &RefinementSteps,
                                               bool &UseOneConstNR,
                                               bool Reciprocal) const {
   EVT VT = Operand.getValueType();
 
   if (VT == MVT::f32) {
     RefinementSteps = 0;
     return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
   }
 
   // TODO: There is also f64 rsq instruction, but the documentation is less
   // clear on its precision.
 
   return SDValue();
 }
 
 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
                                                SelectionDAG &DAG, int Enabled,
                                                int &RefinementSteps) const {
   EVT VT = Operand.getValueType();
 
   if (VT == MVT::f32) {
     // Reciprocal, < 1 ulp error.
     //
     // This reciprocal approximation converges to < 0.5 ulp error with one
     // newton rhapson performed with two fused multiple adds (FMAs).
 
     RefinementSteps = 0;
     return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
   }
 
   // TODO: There is also f64 rcp instruction, but the documentation is less
   // clear on its precision.
 
   return SDValue();
 }
 
 void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
     const SDValue Op, KnownBits &Known,
     const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
 
   Known.resetAll(); // Don't know anything.
 
   unsigned Opc = Op.getOpcode();
 
   switch (Opc) {
   default:
     break;
   case AMDGPUISD::CARRY:
   case AMDGPUISD::BORROW: {
     Known.Zero = APInt::getHighBitsSet(32, 31);
     break;
   }
 
   case AMDGPUISD::BFE_I32:
   case AMDGPUISD::BFE_U32: {
     ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2));
     if (!CWidth)
       return;
 
     uint32_t Width = CWidth->getZExtValue() & 0x1f;
 
     if (Opc == AMDGPUISD::BFE_U32)
       Known.Zero = APInt::getHighBitsSet(32, 32 - Width);
 
     break;
   }
   case AMDGPUISD::FP_TO_FP16:
   case AMDGPUISD::FP16_ZEXT: {
     unsigned BitWidth = Known.getBitWidth();
 
     // High bits are zero.
     Known.Zero = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
     break;
   }
   case AMDGPUISD::MUL_U24:
   case AMDGPUISD::MUL_I24: {
     KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
     KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
     unsigned TrailZ = LHSKnown.countMinTrailingZeros() +
                       RHSKnown.countMinTrailingZeros();
     Known.Zero.setLowBits(std::min(TrailZ, 32u));
     // Skip extra check if all bits are known zeros.
     if (TrailZ >= 32)
       break;
 
     // Truncate to 24 bits.
     LHSKnown = LHSKnown.trunc(24);
     RHSKnown = RHSKnown.trunc(24);
 
     if (Opc == AMDGPUISD::MUL_I24) {
       unsigned LHSValBits = 24 - LHSKnown.countMinSignBits();
       unsigned RHSValBits = 24 - RHSKnown.countMinSignBits();
       unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
       if (MaxValBits >= 32)
         break;
       bool LHSNegative = LHSKnown.isNegative();
       bool LHSNonNegative = LHSKnown.isNonNegative();
       bool LHSPositive = LHSKnown.isStrictlyPositive();
       bool RHSNegative = RHSKnown.isNegative();
       bool RHSNonNegative = RHSKnown.isNonNegative();
       bool RHSPositive = RHSKnown.isStrictlyPositive();
 
       if ((LHSNonNegative && RHSNonNegative) || (LHSNegative && RHSNegative))
         Known.Zero.setHighBits(32 - MaxValBits);
       else if ((LHSNegative && RHSPositive) || (LHSPositive && RHSNegative))
         Known.One.setHighBits(32 - MaxValBits);
     } else {
       unsigned LHSValBits = 24 - LHSKnown.countMinLeadingZeros();
       unsigned RHSValBits = 24 - RHSKnown.countMinLeadingZeros();
       unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
       if (MaxValBits >= 32)
         break;
       Known.Zero.setHighBits(32 - MaxValBits);
     }
     break;
   }
   case AMDGPUISD::PERM: {
     ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Op.getOperand(2));
     if (!CMask)
       return;
 
     KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
     KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
     unsigned Sel = CMask->getZExtValue();
 
     for (unsigned I = 0; I < 32; I += 8) {
       unsigned SelBits = Sel & 0xff;
       if (SelBits < 4) {
         SelBits *= 8;
         Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
         Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
       } else if (SelBits < 7) {
         SelBits = (SelBits & 3) * 8;
         Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
         Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
       } else if (SelBits == 0x0c) {
         Known.Zero |= 0xFFull << I;
       } else if (SelBits > 0x0c) {
         Known.One |= 0xFFull << I;
       }
       Sel >>= 8;
     }
     break;
   }
   case AMDGPUISD::BUFFER_LOAD_UBYTE:  {
     Known.Zero.setHighBits(24);
     break;
   }
   case AMDGPUISD::BUFFER_LOAD_USHORT: {
     Known.Zero.setHighBits(16);
     break;
   }
   case AMDGPUISD::LDS: {
     auto GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
-    unsigned Align = GA->getGlobal()->getAlignment();
+    Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout());
 
     Known.Zero.setHighBits(16);
-    if (Align)
-      Known.Zero.setLowBits(Log2_32(Align));
+    Known.Zero.setLowBits(Log2(Alignment));
     break;
   }
   case ISD::INTRINSIC_WO_CHAIN: {
     unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
     switch (IID) {
     case Intrinsic::amdgcn_mbcnt_lo:
     case Intrinsic::amdgcn_mbcnt_hi: {
       const GCNSubtarget &ST =

llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
     // TODO: We could emit code to handle the initialization somewhere.
     if (!AMDGPUTargetLowering::hasDefinedInitializer(GV)) {
       const SITargetLowering *TLI = ST.getTargetLowering();
       if (!TLI->shouldUseLDSConstAddress(GV)) {
         MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
         return true; // Leave in place;
       }
 
-      B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(), *GV));
+      B.buildConstant(
+          DstReg,
+          MFI->allocateLDSGlobal(B.getDataLayout(), *cast<GlobalVariable>(GV)));
       MI.eraseFromParent();
       return true;
     }
 
     const Function &Fn = MF.getFunction();
     DiagnosticInfoUnsupported BadInit(
       Fn, "unsupported initializer for address space", MI.getDebugLoc());
     Fn.getContext().diagnose(BadInit);

llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h

public:
   bool isMemoryBound() const {
     return MemoryBound;
   }
 
   bool needsWaveLimiter() const {
     return WaveLimiter;
   }
 
-  unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalValue &GV);
+  unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV);
 };
 
 }
 #endif

llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp

WaveLimiter = WaveLimitAttr.isStringAttribute() &&
                 WaveLimitAttr.getValueAsString() == "true";
 
   CallingConv::ID CC = F.getCallingConv();
   if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL)
     ExplicitKernArgSize = ST.getExplicitKernArgSize(F, MaxKernArgAlign);
 }
 
 unsigned AMDGPUMachineFunction::allocateLDSGlobal(const DataLayout &DL,
-                                                  const GlobalValue &GV) {
+                                                  const GlobalVariable &GV) {
   auto Entry = LocalMemoryObjects.insert(std::make_pair(&GV, 0));
   if (!Entry.second)
     return Entry.first->second;
 
   unsigned Align = GV.getAlignment();
   if (Align == 0)
     Align = DL.getABITypeAlignment(GV.getValueType());
 

llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp

case PPC::LWA: {
     // Verify alignment is legal, so we don't create relocations
     // that can't be supported.
     // FIXME:  This test is currently disabled for Darwin.  The test
     // suite shows a handful of test cases that fail this check for
     // Darwin.  Those need to be investigated before this sanity test
     // can be enabled for those subtargets.
     unsigned OpNum = (MI->getOpcode() == PPC::STD) ? 2 : 1;
     const MachineOperand &MO = MI->getOperand(OpNum);
-    if (MO.isGlobal() && MO.getGlobal()->getAlignment() < 4)
+    if (MO.isGlobal()) {
+      const DataLayout &DL = MO.getGlobal()->getParent()->getDataLayout();
+      if (MO.getGlobal()->getPointerAlignment(DL) < 4)
         llvm_unreachable("Global must be word-aligned for LD, STD, LWA!");
+    }
     // Now process the instruction normally.
     break;
   }
   }
 
   LowerPPCMachineInstrToMCInst(MI, TmpInst, *this);
   EmitToStreamer(*OutStreamer, TmpInst);
 }

llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp

while (Position != CurDAG->allnodes_begin()) {
 
     // On PPC64, the TOC base pointer is guaranteed by the ABI only to have
     // 8-byte alignment, and so we can only use offsets less than 8 (otherwise,
     // we might have needed different @ha relocation values for the offset
     // pointers).
     int MaxDisplacement = 7;
     if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
       const GlobalValue *GV = GA->getGlobal();
-      MaxDisplacement = std::min((int) GV->getAlignment() - 1, MaxDisplacement);
+      Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout());
+      MaxDisplacement = std::min((int)Alignment.value() - 1, MaxDisplacement);
     }
 
     bool UpdateHBase = false;
     SDValue HBase = Base.getOperand(0);
 
     int Offset = N->getConstantOperandVal(FirstOp);
     if (ReplaceFlags) {
       if (Offset < 0 || Offset > MaxDisplacement) {
while (Position != CurDAG->allnodes_begin()) {
     LLVM_DEBUG(dbgs() << "\n");
 
     // If the relocation information isn't already present on the
     // immediate operand, add it now.
     if (ReplaceFlags) {
       if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(ImmOpnd)) {
         SDLoc dl(GA);
         const GlobalValue *GV = GA->getGlobal();
+        Align Alignment = GV->getPointerAlignment(CurDAG->getDataLayout());
         // We can't perform this optimization for data whose alignment
         // is insufficient for the instruction encoding.
-        if (GV->getAlignment() < 4 &&
-            (RequiresMod4Offset || (Offset % 4) != 0)) {
+        if (Alignment < 4 && (RequiresMod4Offset || (Offset % 4) != 0)) {
           LLVM_DEBUG(dbgs() << "Rejected this candidate for alignment.\n\n");
           continue;
         }
         ImmOpnd = CurDAG->getTargetGlobalAddress(GV, dl, MVT::i64, Offset, Flags);
       } else if (ConstantPoolSDNode *CP =
                  dyn_cast<ConstantPoolSDNode>(ImmOpnd)) {
         const Constant *C = CP->getConstVal();
         ImmOpnd = CurDAG->getTargetConstantPool(C, MVT::i64, CP->getAlign(),

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

if (DefMI.getOpcode() == PPC::ADDItocL) {
     if (III.ImmMustBeMultipleOf > 4 ||
        III.TruncateImmTo || III.ImmWidth != 16)
       return false;
 
     // Going from XForm to DForm loads means that the displacement needs to be
     // not just an immediate but also a multiple of 4, or 16 depending on the
     // load. A DForm load cannot be represented if it is a multiple of say 2.
     // XForm loads do not have this restriction.
-    if (ImmMO.isGlobal() &&
-        ImmMO.getGlobal()->getAlignment() < III.ImmMustBeMultipleOf)
+    if (ImmMO.isGlobal()) {
+      const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
+      if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
         return false;
+    }
 
     return true;
   }
 
   if (ImmMO.isImm()) {
     // It is Imm, we need to check if the Imm fit the range.
     int64_t Immediate = ImmMO.getImm();
     // Sign-extend to 64-bits.

llvm/lib/Target/SystemZ/SystemZSubtarget.cpp

 
 
 bool SystemZSubtarget::enableSubRegLiveness() const {
   return UseSubRegLiveness;
 }
 
 bool SystemZSubtarget::isPC32DBLSymbol(const GlobalValue *GV,
                                        CodeModel::Model CM) const {
-  // PC32DBL accesses require the low bit to be clear.  Note that a zero
-  // value selects the default alignment and is therefore OK.
-  if (GV->getAlignment() == 1)
+  // PC32DBL accesses require the low bit to be clear.
+  //
+  // FIXME: Explicitly check for functions: the datalayout is currently
+  // missing information about function pointers.
+  const DataLayout &DL = GV->getParent()->getDataLayout();
+  if (GV->getPointerAlignment(DL) == 1 && !GV->getValueType()->isFunctionTy())
     return false;
 
   // For the small model, all locally-binding symbols are in range.
   if (CM == CodeModel::Small)
     return TLInfo.getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
 
   // For Medium and above, assume that the symbol is not within the 4GB range.
   // Taking the address of locally-defined text would be OK, but that
   // case isn't easy to detect.
   return false;
 }

llvm/lib/Target/XCore/XCoreISelLowering.cpp

if (!LD->isVolatile()) {
     if (DAG.isBaseWithConstantOffset(BasePtr) &&
         isWordAligned(BasePtr->getOperand(0), DAG)) {
       SDValue NewBasePtr = BasePtr->getOperand(0);
       Offset = cast<ConstantSDNode>(BasePtr->getOperand(1))->getSExtValue();
       return lowerLoadWordFromAlignedBasePlusOffset(DL, Chain, NewBasePtr,
                                                     Offset, DAG);
     }
     if (TLI.isGAPlusOffset(BasePtr.getNode(), GV, Offset) &&
-        MinAlign(GV->getAlignment(), 4) == 4) {
+        GV->getPointerAlignment(DAG.getDataLayout()) >= 4) {
       SDValue NewBasePtr = DAG.getGlobalAddress(GV, DL,
                                                 BasePtr->getValueType(0));
       return lowerLoadWordFromAlignedBasePlusOffset(DL, Chain, NewBasePtr,
                                                     Offset, DAG);
     }
   }
 
   if (LD->getAlignment() == 2) {

llvm/test/CodeGen/AArch64/funcptr_cast.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s
 
 define i8 @test() {
-; CHECK-LABEL: @test
-; CHECK: adrp {{x[0-9]+}}, foo
-; CHECK: add {{x[0-9]+}}, {{x[0-9]+}}, :lo12:foo
-; CHECK: ldrb w0, [{{x[0-9]+}}]
+; CHECK-LABEL: test:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adrp x8, foo
+; CHECK-NEXT:    ldrb w0, [x8, :lo12:foo]
+; CHECK-NEXT:    ret
 entry:
   %0 = load i8, i8* bitcast (void (...)* @foo to i8*), align 1
   ret i8 %0
 }
 
 declare void @foo(...)

llvm/test/CodeGen/AArch64/global-alignment.ll

; CHECK: ldr x0, [x[[HIBITS]], {{#?}}:lo12:var32_align64]
 
   ret i64 %val
 }
 
 define i64 @test_var32_alias() {
 ; CHECK-LABEL: test_var32_alias:
   %addr = bitcast [3 x i32]* @alias to i64*
 
-  ; Test that we can find the alignment for aliases.
+  ; We don't know anything about the alignment of aliases.
   %val = load i64, i64* %addr
 ; CHECK: adrp x[[HIBITS:[0-9]+]], alias
-; CHECK-NOT: add x[[HIBITS]]
-; CHECK: ldr x0, [x[[HIBITS]], {{#?}}:lo12:alias]
+; CHECK: add x[[ADDR:[0-9]+]], x[[HIBITS]], {{#?}}:lo12:alias
+; CHECK: ldr x0, [x[[ADDR]]]
 
   ret i64 %val
 }
 
 @yet_another_var = external global {i32, i32}
 
 define i64 @test_yet_another_var() {
 ; CHECK-LABEL: test_yet_another_var:

llvm/test/CodeGen/PowerPC/atomics-constant.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
 
 target triple = "powerpc64le-unknown-linux-gnu"
 
 @a = constant i64 zeroinitializer
 
 define i64 @foo() {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis 3, 2, a@toc@ha
 ; CHECK-NEXT:    li 4, 0
-; CHECK-NEXT:    addi 3, 3, a@toc@l
+; CHECK-NEXT:    addis 3, 2, a@toc@ha
+; CHECK-NEXT:    ld 3, a@toc@l(3)
 ; CHECK-NEXT:    cmpd 7, 4, 4
-; CHECK-NEXT:    ld 3, 0(3)
 ; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    bne- 7, .+4
 ; CHECK-NEXT:    isync
 ; CHECK-NEXT:    blr
 entry:
   %value = load atomic i64, i64* @a acquire, align 8
   ret i64 %value
 }