[AMDGPU] Add DAG ISel support for preloaded kernel arguments

[kerbowa]

Summary

This patch adds the DAG isel changes for kernel argument preloading.
These changes are not usable with older firmware but subsequent patches
in the series will make the codegen backwards compatible. This patch
should only be submitted alongside that subsequent patch.

Preloading here begins from the start of the kernel arguments until the
amount of arguments indicated by the CL flag
amdgpu-kernarg-preload-count.

Aggregates and arguments passed by-ref are not supported.

Special care for the alignment of the kernarg segment is needed as well
as consideration of the alignment of addressable SGPR tuples when we
cannot directly use misaligned large tuples that the arguments are
loaded to.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp

63

Probably should but I don't even know if this is part of debug printing anywhere, I don't know the last time I saw this

llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

463

Can you move this to hasKernargPreload helper or something? Probably should make it a full subtarget feature on its own

llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp

261–263

Can you separate all the formatting changes out?

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

5003

Check the subtarget for the limit

5003–5004

Is the interaction with .amdhsa_user_sgpr_count checked?

llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp

1949–1955

Can you split the assembler / MC bits into a separate patch?

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp

904–906

Needs some temporary variables

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

2235

I do not understand this metadata system, you know directly from the IR arguments the register layout

2245

unchecked dyn_caast

llvm/test/MC/AMDGPU/hsa-v4.s

113

Need some tests with the limit exceeded diagnostic and interactions with amdhsa_user_sgpr_count

[kerbowa]

Factor out MC changes, cleanup, add nop padding.

[kerbowa]

Rebase with non-metadata based kernarg preload tracking.

[kerbowa]

Fixup feature checks.

[kerbowa]

Limit preloading to gfx940.

[arsenm]

Also should do the globalisel equivalent

llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h

159

Why is it a map? Isn't this just an array?

llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

223

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp

834–840

Shouldn't need this, there are already nop emission utilities?

846

emitValueToAlignment?

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

2235

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

245

This can return ArrayRef?

266

Typo Kerarg

llvm/test/CodeGen/AMDGPU/preload-kernargs.ll

21 ↗

(On Diff #556812)

should be able to use a directive to avoid spamming this

[kerbowa]

Rebase, address comments.

llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h

159

We would at least need an offset stored somewhere since we shouldn't assume preloads will always start at the first kernel argument.

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp

846

I'm not sure how relevant the alignment is here actually. What matters is that we are emitting exactly 256 bytes.

[bcahoon]

Looks good to me. I also tried some additional tests and they also worked as expected.

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

2307

Always

[kerbowa]

Fix case where aligned register class was not a trivially lowerable copy.

[bcahoon]

Changes LGTM. Thanks!

llvm/include/llvm/Support/AMDHSAKernelDescriptor.h

enum : int32_t {
   KERNEL_CODE_PROPERTY(ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, 6, 1),
   KERNEL_CODE_PROPERTY(RESERVED0, 7, 3),
   KERNEL_CODE_PROPERTY(ENABLE_WAVEFRONT_SIZE32, 10, 1), // GFX10+
   KERNEL_CODE_PROPERTY(USES_DYNAMIC_STACK, 11, 1),
   KERNEL_CODE_PROPERTY(RESERVED1, 12, 4),
 };
 #undef KERNEL_CODE_PROPERTY
 
+// Kernarg preload specification.
+#define KERNARG_PRELOAD_SPEC(NAME, SHIFT, WIDTH)                               \
+  AMDHSA_BITS_ENUM_ENTRY(KERNARG_PRELOAD_SPEC_##NAME, SHIFT, WIDTH)
+enum : int32_t {
+  KERNARG_PRELOAD_SPEC(LENGTH, 0, 7),
+  KERNARG_PRELOAD_SPEC(OFFSET, 7, 9),
+};
+#undef KERNARG_PRELOAD_SPEC
+
 // Kernel descriptor. Must be kept backwards compatible.
 struct kernel_descriptor_t {
   uint32_t group_segment_fixed_size;
   uint32_t private_segment_fixed_size;
   uint32_t kernarg_size;
   uint8_t reserved0[4];
   int64_t kernel_code_entry_byte_offset;
   uint8_t reserved1[20];
   uint32_t compute_pgm_rsrc3; // GFX10+ and GFX90A+
   uint32_t compute_pgm_rsrc1;
   uint32_t compute_pgm_rsrc2;
   uint16_t kernel_code_properties;
-  uint8_t reserved2[6];
+  uint16_t kernarg_preload;
+  uint8_t reserved3[4];
 };
 
 enum : uint32_t {
   GROUP_SEGMENT_FIXED_SIZE_OFFSET = 0,
   PRIVATE_SEGMENT_FIXED_SIZE_OFFSET = 4,
   KERNARG_SIZE_OFFSET = 8,
   RESERVED0_OFFSET = 12,
   KERNEL_CODE_ENTRY_BYTE_OFFSET_OFFSET = 16,
   RESERVED1_OFFSET = 24,
   COMPUTE_PGM_RSRC3_OFFSET = 44,
   COMPUTE_PGM_RSRC1_OFFSET = 48,
   COMPUTE_PGM_RSRC2_OFFSET = 52,
   KERNEL_CODE_PROPERTIES_OFFSET = 56,
-  RESERVED2_OFFSET = 58,
+  KERNARG_PRELOAD_OFFSET = 58,
+  RESERVED3_OFFSET = 60
 };
 
 static_assert(
     sizeof(kernel_descriptor_t) == 64,
     "invalid size for kernel_descriptor_t");
 static_assert(offsetof(kernel_descriptor_t, group_segment_fixed_size) ==
                   GROUP_SEGMENT_FIXED_SIZE_OFFSET,
               "invalid offset for group_segment_fixed_size");
static_assert(offsetof(kernel_descriptor_t, compute_pgm_rsrc1) ==
                   COMPUTE_PGM_RSRC1_OFFSET,
               "invalid offset for compute_pgm_rsrc1");
 static_assert(offsetof(kernel_descriptor_t, compute_pgm_rsrc2) ==
                   COMPUTE_PGM_RSRC2_OFFSET,
               "invalid offset for compute_pgm_rsrc2");
 static_assert(offsetof(kernel_descriptor_t, kernel_code_properties) ==
                   KERNEL_CODE_PROPERTIES_OFFSET,
               "invalid offset for kernel_code_properties");
-static_assert(offsetof(kernel_descriptor_t, reserved2) == RESERVED2_OFFSET,
-              "invalid offset for reserved2");
+static_assert(offsetof(kernel_descriptor_t, kernarg_preload) ==
+                  KERNARG_PRELOAD_OFFSET,
+              "invalid offset for kernarg_preload");
+static_assert(offsetof(kernel_descriptor_t, reserved3) == RESERVED3_OFFSET,
+              "invalid offset for reserved3");
 
 } // end namespace amdhsa
 } // end namespace llvm
 
 #endif // LLVM_SUPPORT_AMDHSAKERNELDESCRIPTOR_H

llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h

 //==- AMDGPUArgumentrUsageInfo.h - Function Arg Usage Info -------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUARGUMENTUSAGEINFO_H
 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUARGUMENTUSAGEINFO_H
 
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/Register.h"
+#include "llvm/MC/MCRegister.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
 
 class Function;
 class LLT;
 class raw_ostream;
 class TargetRegisterClass;
private:
 
   // Bitmask to locate argument within the register.
   unsigned Mask;
 
   bool IsStack : 1;
   bool IsSet : 1;
 
 public:
-  constexpr ArgDescriptor(unsigned Val = 0, unsigned Mask = ~0u,
-                bool IsStack = false, bool IsSet = false)
+  ArgDescriptor(unsigned Val = 0, unsigned Mask = ~0u, bool IsStack = false,
+                bool IsSet = false)
       : Reg(Val), Mask(Mask), IsStack(IsStack), IsSet(IsSet) {}
 
-  static constexpr ArgDescriptor createRegister(Register Reg,
-                                                unsigned Mask = ~0u) {
+  static ArgDescriptor createRegister(Register Reg, unsigned Mask = ~0u) {
     return ArgDescriptor(Reg, Mask, false, true);
   }
 
-  static constexpr ArgDescriptor createStack(unsigned Offset,
-                                             unsigned Mask = ~0u) {
+  static ArgDescriptor createStack(unsigned Offset, unsigned Mask = ~0u) {
     return ArgDescriptor(Offset, Mask, true, true);
   }
 
-  static constexpr ArgDescriptor createArg(const ArgDescriptor &Arg,
-                                           unsigned Mask) {
+  static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask) {
     return ArgDescriptor(Arg.Reg, Mask, Arg.IsStack, Arg.IsSet);
   }
 
   bool isSet() const {
     return IsSet;
   }
 
   explicit operator bool() const {
public:
   void print(raw_ostream &OS, const TargetRegisterInfo *TRI = nullptr) const;
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const ArgDescriptor &Arg) {
   Arg.print(OS);
   return OS;
 }
 
+struct KernArgPreloadDescriptor : public ArgDescriptor {
+  KernArgPreloadDescriptor() {}
+  SmallVector<MCRegister> Regs;
+};
+
 struct AMDGPUFunctionArgInfo {
   enum PreloadedValue {
     // SGPRS:
     PRIVATE_SEGMENT_BUFFER = 0,
     DISPATCH_PTR        =  1,
     QUEUE_PTR           =  2,
     KERNARG_SEGMENT_PTR =  3,
     DISPATCH_ID         =  4,
struct AMDGPUFunctionArgInfo {
   ArgDescriptor ImplicitBufferPtr;
 
   // VGPRs inputs. For entry functions these are either v0, v1 and v2 or packed
   // into v0, 10 bits per dimension if packed-tid is set.
   ArgDescriptor WorkItemIDX;
   ArgDescriptor WorkItemIDY;
   ArgDescriptor WorkItemIDZ;
 
+  // Map the index of preloaded kernel arguments to its descriptor.
+  SmallDenseMap<int, KernArgPreloadDescriptor> PreloadKernArgs{};
+
   std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
   getPreloadedValue(PreloadedValue Value) const;
 
-  static constexpr AMDGPUFunctionArgInfo fixedABILayout();
+  static AMDGPUFunctionArgInfo fixedABILayout();
 };
 
 class AMDGPUArgumentUsageInfo : public ImmutablePass {
 private:
   DenseMap<const Function *, AMDGPUFunctionArgInfo> ArgInfoMap;
 
 public:
   static char ID;

llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp

bool AMDGPUArgumentUsageInfo::doInitialization(Module &M) {
   return false;
 }
 
 bool AMDGPUArgumentUsageInfo::doFinalization(Module &M) {
   ArgInfoMap.clear();
   return false;
 }
 
+// TODO: Print preload kernargs?
 void AMDGPUArgumentUsageInfo::print(raw_ostream &OS, const Module *M) const {
   for (const auto &FI : ArgInfoMap) {
     OS << "Arguments for " << FI.first->getName() << '\n'
        << "  PrivateSegmentBuffer: " << FI.second.PrivateSegmentBuffer
        << "  DispatchPtr: " << FI.second.DispatchPtr
        << "  QueuePtr: " << FI.second.QueuePtr
        << "  KernargSegmentPtr: " << FI.second.KernargSegmentPtr
        << "  DispatchID: " << FI.second.DispatchID
return std::tuple(WorkItemIDY ? &WorkItemIDY : nullptr,
                       &AMDGPU::VGPR_32RegClass, LLT::scalar(32));
   case AMDGPUFunctionArgInfo::WORKITEM_ID_Z:
     return std::tuple(WorkItemIDZ ? &WorkItemIDZ : nullptr,
                       &AMDGPU::VGPR_32RegClass, LLT::scalar(32));
   }
   llvm_unreachable("unexpected preloaded value type");
 }
 
-constexpr AMDGPUFunctionArgInfo AMDGPUFunctionArgInfo::fixedABILayout() {
+AMDGPUFunctionArgInfo AMDGPUFunctionArgInfo::fixedABILayout() {
   AMDGPUFunctionArgInfo AI;
   AI.PrivateSegmentBuffer
     = ArgDescriptor::createRegister(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3);
   AI.DispatchPtr = ArgDescriptor::createRegister(AMDGPU::SGPR4_SGPR5);
   AI.QueuePtr = ArgDescriptor::createRegister(AMDGPU::SGPR6_SGPR7);
 
   // Do not pass kernarg segment pointer, only pass increment version in its
   // place.

llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

if ((STM.isMesaKernel(F) || CodeObjectVersion == AMDGPU::AMDHSA_COV2) &&
     amd_kernel_code_t KernelCode;
     getAmdKernelCode(KernelCode, CurrentProgramInfo, *MF);
     getTargetStreamer()->EmitAMDKernelCodeT(KernelCode);
   }
 
   if (STM.isAmdHsaOS())
     HSAMetadataStream->emitKernel(*MF, CurrentProgramInfo);
 }
 
 void AMDGPUAsmPrinter::emitFunctionBodyEnd() {
   const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
   if (!MFI.isEntryFunction())
     return;
 
   if (TM.getTargetTriple().getOS() != Triple::AMDHSA ||
       CodeObjectVersion == AMDGPU::AMDHSA_COV2)
     return;
uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
   return KernelCodeProperties;
 }
 
 amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(
     const MachineFunction &MF,
     const SIProgramInfo &PI) const {
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   const Function &F = MF.getFunction();
+  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
   amdhsa::kernel_descriptor_t KernelDescriptor;
   memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor));
 
   assert(isUInt<32>(PI.ScratchSize));
   assert(isUInt<32>(PI.getComputePGMRSrc1()));
   assert(isUInt<32>(PI.getComputePGMRSrc2()));
 
   KernelDescriptor.group_segment_fixed_size = PI.LDSSize;
   KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
 
   Align MaxKernArgAlign;
   KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign);
 
   KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1();
   KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2();
   KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
 
   assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0);
   if (STM.hasGFX90AInsts())
     KernelDescriptor.compute_pgm_rsrc3 =
       CurrentProgramInfo.ComputePGMRSrc3GFX90A;
 
+  if (STM.hasGFX90AInsts())
+    KernelDescriptor.kernarg_preload =
+        static_cast<uint16_t>(Info->getNumKernargPreloadedSGPRs());
+
   return KernelDescriptor;
 }
 
 bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   // Init target streamer lazily on the first function so that previous passes
   // can set metadata.
   if (!IsTargetStreamerInitialized)
     initTargetStreamer(*MF.getFunction().getParent());

llvm/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp

public:
   // arguments.
   void setInitialFreeUserSGPRsCount() {
     const unsigned MaxUserSGRPs = ST.getMaxNumUserSGPRs();
     GCNUserSGPRUsageInfo UserSGPRInfo(F, ST);
 
     NumFreeUserSGPRs = MaxUserSGRPs - UserSGPRInfo.getNumUsedUserSGPRs();
   }
 
-  unsigned allocPreloadSGPRs(bool IsInReg, bool InPreloadSequence,
-                             unsigned AllocSize, uint64_t ArgOffset,
+  unsigned allocPreloadSGPRs(unsigned AllocSize, uint64_t ArgOffset,
                              uint64_t LastExplicitArgOffset) {
-
-    if (!IsInReg || !InPreloadSequence)
-      return 0;
-
     //  Check if this arguemnt may be loaded into the same register as the
     //  previous argument.
     if (!isAligned(Align(4), ArgOffset) && AllocSize < 4)
       return 1;
 
     // Pad SGPRs for kernarg alignment.
     unsigned Padding = ArgOffset - LastExplicitArgOffset;
     unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
for (Argument &Arg : F.args()) {
     ExplicitArgOffset = alignTo(ExplicitArgOffset, ABITypeAlign) + AllocSize;
 
     if (Arg.use_empty()) {
       InPreloadSequence = false;
       continue;
     }
 
     // Try to preload this argument.
-    unsigned PreloadSGPRs = PreloadInfo.allocPreloadSGPRs(
-        Arg.hasInRegAttr(), InPreloadSequence, AllocSize, EltOffset,
-        LastExplicitArgOffset);
-    if (PreloadSGPRs && !Arg.getType()->isAggregateType()) {
+    if (Arg.hasInRegAttr() && InPreloadSequence && ST.hasGFX90AInsts() &&
+        !Arg.getType()->isAggregateType()) {
+      if (unsigned PreloadSGPRs = PreloadInfo.allocPreloadSGPRs(
+              AllocSize, EltOffset, LastExplicitArgOffset)) {
         // Preload this argument.
         HasPreloadArgs = true;
         MDBuilder MDB(Ctx);
-      auto *MDIndex =
-          MDB.createConstant(llvm::ConstantInt::get(Builder.getInt32Ty(), Idx));
+        auto *MDIndex = MDB.createConstant(
+            llvm::ConstantInt::get(Builder.getInt32Ty(), Idx));
         auto *MDAllocSizeSGPRs = MDB.createConstant(
             llvm::ConstantInt::get(Builder.getInt32Ty(), PreloadSGPRs));
         PreloadInfo.KernelArgMetadata.push_back(
             llvm::MDNode::get(Ctx, {MDIndex, MDAllocSizeSGPRs}));
-    } else {
-      InPreloadSequence = false;
+        continue;
       }
+    }
+    InPreloadSequence = false;
 
-    // If this is byval, the loads are already explicit in the function. We just
-    // need to rewrite the pointer values.
+    // If this is byval, the loads are already explicit in the function. We
+    // just need to rewrite the pointer values.
     if (IsByRef) {
       Value *ArgOffsetPtr = Builder.CreateConstInBoundsGEP1_64(
           Builder.getInt8Ty(), KernArgSegment, EltOffset,
           Arg.getName() + ".byval.kernarg.offset");
 
       Value *CastOffsetPtr =
           Builder.CreateAddrSpaceCast(ArgOffsetPtr, Arg.getType());
       Arg.replaceAllUsesWith(CastOffsetPtr);
       continue;
     }
 
     if (PointerType *PT = dyn_cast<PointerType>(ArgTy)) {
       // FIXME: Hack. We rely on AssertZext to be able to fold DS addressing
-      // modes on SI to know the high bits are 0 so pointer adds don't wrap. We
-      // can't represent this with range metadata because it's only allowed for
-      // integer types.
+      // modes on SI to know the high bits are 0 so pointer adds don't wrap.
+      // We can't represent this with range metadata because it's only allowed
+      // for integer types.
       if ((PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
            PT->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) &&
           !ST.hasUsableDSOffset())
         continue;
 
       // FIXME: We can replace this with equivalent alias.scope/noalias
       // metadata, but this appears to be a lot of work.
       if (Arg.hasNoAliasAttr())
for (Argument &Arg : F.args()) {
     int64_t OffsetDiff = EltOffset - AlignDownOffset;
     Align AdjustedAlign = commonAlignment(
         KernArgBaseAlign, DoShiftOpt ? AlignDownOffset : EltOffset);
 
     Value *ArgPtr;
     Type *AdjustedArgTy;
     if (DoShiftOpt) { // FIXME: Handle aggregate types
       // Since we don't have sub-dword scalar loads, avoid doing an extload by
-      // loading earlier than the argument address, and extracting the relevant
-      // bits.
+      // loading earlier than the argument address, and extracting the
+      // relevant bits.
       //
-      // Additionally widen any sub-dword load to i32 even if suitably aligned,
-      // so that CSE between different argument loads works easily.
+      // Additionally widen any sub-dword load to i32 even if suitably
+      // aligned, so that CSE between different argument loads works easily.
       ArgPtr = Builder.CreateConstInBoundsGEP1_64(
           Builder.getInt8Ty(), KernArgSegment, AlignDownOffset,
           Arg.getName() + ".kernarg.offset.align.down");
       AdjustedArgTy = Builder.getInt32Ty();
     } else {
       ArgPtr = Builder.CreateConstInBoundsGEP1_64(
           Builder.getInt8Ty(), KernArgSegment, EltOffset,
           Arg.getName() + ".kernarg.offset");
for (Argument &Arg : F.args()) {
 
     if (isa<PointerType>(ArgTy)) {
       if (Arg.hasNonNullAttr())
         Load->setMetadata(LLVMContext::MD_nonnull, MDNode::get(Ctx, {}));
 
       uint64_t DerefBytes = Arg.getDereferenceableBytes();
       if (DerefBytes != 0) {
         Load->setMetadata(
             LLVMContext::MD_dereferenceable,
-          MDNode::get(Ctx,
-                      MDB.createConstant(
-                        ConstantInt::get(Builder.getInt64Ty(), DerefBytes))));
+            MDNode::get(Ctx, MDB.createConstant(ConstantInt::get(
+                                 Builder.getInt64Ty(), DerefBytes))));
       }
 
       uint64_t DerefOrNullBytes = Arg.getDereferenceableOrNullBytes();
       if (DerefOrNullBytes != 0) {
         Load->setMetadata(
             LLVMContext::MD_dereferenceable_or_null,
-          MDNode::get(Ctx,
-                      MDB.createConstant(ConstantInt::get(Builder.getInt64Ty(),
-                                                          DerefOrNullBytes))));
+            MDNode::get(Ctx, MDB.createConstant(ConstantInt::get(
+                                 Builder.getInt64Ty(), DerefOrNullBytes))));
       }
 
       if (MaybeAlign ParamAlign = Arg.getParamAlign()) {
         Load->setMetadata(
             LLVMContext::MD_align,
             MDNode::get(Ctx, MDB.createConstant(ConstantInt::get(
                                  Builder.getInt64Ty(), ParamAlign->value()))));
       }
     }
 
     // TODO: Convert noalias arg to !noalias
 
     if (DoShiftOpt) {
-      Value *ExtractBits = OffsetDiff == 0 ?
-        Load : Builder.CreateLShr(Load, OffsetDiff * 8);
+      Value *ExtractBits =
+          OffsetDiff == 0 ? Load : Builder.CreateLShr(Load, OffsetDiff * 8);
 
       IntegerType *ArgIntTy = Builder.getIntNTy(Size);
       Value *Trunc = Builder.CreateTrunc(ExtractBits, ArgIntTy);
-      Value *NewVal = Builder.CreateBitCast(Trunc, ArgTy,
-                                            Arg.getName() + ".load");
+      Value *NewVal =
+          Builder.CreateBitCast(Trunc, ArgTy, Arg.getName() + ".load");
       Arg.replaceAllUsesWith(NewVal);
     } else if (IsV3) {
       Value *Shuf = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 2},
                                                 Arg.getName() + ".load");
       Arg.replaceAllUsesWith(Shuf);
     } else {
       Load->setName(Arg.getName() + ".load");
       Arg.replaceAllUsesWith(Load);

llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

if (ID == ".amdhsa_group_segment_fixed_size") {
         return Error(IDRange.Start,
                      "directive is not supported with architected flat scratch",
                      IDRange);
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
                        Val, ValRange);
       if (Val)
         ImpliedUserSGPRCount += 4;
+    } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") {
+      if (Val > 16)
+        return OutOfRangeError(ValRange);
+      PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, Val,
+                       ValRange);
+    } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") {
+      if (Val >= 1024)
+        return OutOfRangeError(ValRange);
+      PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, Val,
+                       ValRange);
     } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
       PARSE_BITS_ENTRY(KD.kernel_code_properties,
                        KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val,
                        ValRange);
       if (Val)
         ImpliedUserSGPRCount += 2;
     } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
       PARSE_BITS_ENTRY(KD.kernel_code_properties,

llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp

if (AMDGPU::getAmdhsaCodeObjectVersion() >= AMDGPU::AMDHSA_COV5)
       PRINT_DIRECTIVE(".amdhsa_uses_dynamic_stack",
                       KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK);
 
     if (TwoByteBuffer & KERNEL_CODE_PROPERTY_RESERVED1)
       return MCDisassembler::Fail;
 
     return MCDisassembler::Success;
 
-  case amdhsa::RESERVED2_OFFSET:
-    // 6 bytes from here are reserved, must be 0.
-    ReservedBytes = DE.getBytes(Cursor, 6);
-    for (int I = 0; I < 6; ++I) {
+  case amdhsa::KERNARG_PRELOAD_OFFSET:
+    using namespace amdhsa;
+    TwoByteBuffer = DE.getU16(Cursor);
+    PRINT_DIRECTIVE(".amdhsa_user_sgpr_kernarg_preload_length",
+                    KERNARG_PRELOAD_SPEC_LENGTH);
+    PRINT_DIRECTIVE(".amdhsa_user_sgpr_kernarg_preload_offset",
+                    KERNARG_PRELOAD_SPEC_OFFSET);
+    return MCDisassembler::Success;
+
+  case amdhsa::RESERVED3_OFFSET:
+    // 4 bytes from here are reserved, must be 0.
+    ReservedBytes = DE.getBytes(Cursor, 4);
+    for (int I = 0; I < 4; ++I) {
       if (ReservedBytes[I] != 0)
         return MCDisassembler::Fail;
     }
     return MCDisassembler::Success;
 
   default:
     llvm_unreachable("Unhandled index. Case statements cover everything.");
     return MCDisassembler::Fail;

llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp

PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD,
               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR);
   PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD,
               kernel_code_properties,
               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID);
   if (!hasArchitectedFlatScratch(STI))
     PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD,
                 kernel_code_properties,
                 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT);
+  PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD,
+              kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH);
+  PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD,
+              kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET);
   PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD,
               kernel_code_properties,
               amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE);
   if (IVersion.Major >= 10)
     PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD,
                 kernel_code_properties,
                 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32);
   if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5)
EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_HSA_METADATA,
            [&](MCELFStreamer &OS) {
              OS.emitLabel(DescBegin);
              OS.emitBytes(HSAMetadataString);
              OS.emitLabel(DescEnd);
            });
   return true;
 }
 
 bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
   const uint32_t Encoded_s_code_end = 0xbf9f0000;
   const uint32_t Encoded_s_nop = 0xbf800000;
   uint32_t Encoded_pad = Encoded_s_code_end;
 
   // Instruction cache line size in bytes.
   const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;
   const unsigned CacheLineSize = 1u << Log2CacheLineSize;
 
   // Extra padding amount in bytes to support prefetch mode 3.
   unsigned FillSize = 3 * CacheLineSize;
 
   if (AMDGPU::isGFX90A(STI)) {
     Encoded_pad = Encoded_s_nop;
     FillSize = 16 * CacheLineSize;
   }
 
   MCStreamer &OS = getStreamer();
   OS.pushSection();
   OS.emitValueToAlignment(Align(CacheLineSize), Encoded_pad, 4);
   for (unsigned I = 0; I < FillSize; I += 4)
void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
 
   for (uint8_t Res : KernelDescriptor.reserved0)
     Streamer.emitInt8(Res);
 
   // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The
   // expression being created is:
   //   (start of kernel code) - (start of kernel descriptor)
   // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64.
-  Streamer.emitValue(MCBinaryExpr::createSub(
-      MCSymbolRefExpr::create(
-          KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context),
-      MCSymbolRefExpr::create(
-          KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context),
+  if (KernelDescriptor.kernarg_preload != 0) {
+    Streamer.emitValue(
+        MCBinaryExpr::createSub(
+            MCBinaryExpr::createSub(
+                MCSymbolRefExpr::create(KernelCodeSymbol,
+                                        MCSymbolRefExpr::VK_AMDGPU_REL64,
+                                        Context),
+                MCSymbolRefExpr::create(KernelDescriptorSymbol,
+                                        MCSymbolRefExpr::VK_None, Context),
+                Context),
+            MCConstantExpr::create(0x100, Context), Context),
+        sizeof(KernelDescriptor.kernel_code_entry_byte_offset));
+  } else {
+    Streamer.emitValue(
+        MCBinaryExpr::createSub(
+            MCSymbolRefExpr::create(KernelCodeSymbol,
+                                    MCSymbolRefExpr::VK_AMDGPU_REL64, Context),
+            MCSymbolRefExpr::create(KernelDescriptorSymbol,
+                                    MCSymbolRefExpr::VK_None, Context),
             Context),
         sizeof(KernelDescriptor.kernel_code_entry_byte_offset));
+  }
   for (uint8_t Res : KernelDescriptor.reserved1)
     Streamer.emitInt8(Res);
   Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3);
   Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1);
   Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2);
   Streamer.emitInt16(KernelDescriptor.kernel_code_properties);
-  for (uint8_t Res : KernelDescriptor.reserved2)
+  Streamer.emitInt16(KernelDescriptor.kernarg_preload);
+  for (uint8_t Res : KernelDescriptor.reserved3)
     Streamer.emitInt8(Res);
 }

llvm/lib/Target/AMDGPU/SIISelLowering.h

 //===-- SIISelLowering.h - SI DAG Lowering Interface ------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
 /// SI DAG Lowering interface definition
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
 #define LLVM_LIB_TARGET_AMDGPU_SIISELLOWERING_H
 
-#include "AMDGPUISelLowering.h"
 #include "AMDGPUArgumentUsageInfo.h"
+#include "AMDGPUISelLowering.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFunction.h"
 
 namespace llvm {
 
 class GCNSubtarget;
 class SIMachineFunctionInfo;
 class SIRegisterInfo;
 
bool requiresUniformRegister(MachineFunction &MF,
                                const Value *V) const override;
   Align getPrefLoopAlignment(MachineLoop *ML) const override;
 
   void allocateHSAUserSGPRs(CCState &CCInfo,
                             MachineFunction &MF,
                             const SIRegisterInfo &TRI,
                             SIMachineFunctionInfo &Info) const;
 
+  void allocatePreloadKernArgSGPRs(CCState &CCInfo,
+                                   SmallVectorImpl<CCValAssign> &ArgLocs,
+                                   const SmallVectorImpl<ISD::InputArg> &Ins,
+                                   MachineFunction &MF,
+                                   const SIRegisterInfo &TRI,
+                                   SIMachineFunctionInfo &Info) const;
+
+  void allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
+                           const SIRegisterInfo &TRI,
+                           SIMachineFunctionInfo &Info) const;
+
   void allocateSystemSGPRs(CCState &CCInfo,
                            MachineFunction &MF,
                            SIMachineFunctionInfo &Info,
                            CallingConv::ID CallConv,
                            bool IsShader) const;
 
   void allocateSpecialEntryInputVGPRs(CCState &CCInfo,
                                       MachineFunction &MF,

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

 #include "AMDGPUInstrInfo.h"
 #include "AMDGPUTargetMachine.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIMachineFunctionInfo.h"
 #include "SIRegisterInfo.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/ByteProvider.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/IR/Metadata.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/ModRef.h"
 #include <optional>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "si-lower"
if (MemVT.getStoreSize() < 4 && Alignment < 4) {
 
     SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
     SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
 
     SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
     ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
     ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
 
-
     return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
   }
 
   SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
   SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Alignment,
                              MachineMemOperand::MODereferenceable |
                                  MachineMemOperand::MOInvariant);
 
void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
   }
 
   if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
     Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
     MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
     CCInfo.AllocateReg(FlatScratchInitReg);
   }
 
+  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
+  // these from the dispatch pointer.
+}
+
+void SITargetLowering::allocatePreloadKernArgSGPRs(
+    CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
+    const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
+    const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
+  // Allocate pre-loaded kernel arguemtns.
+  const Function &F = MF.getFunction();
+  MDNode *MD = F.getMetadata("preload_kernel_args");
+  if (!MD)
+    return;
+
+  if (!dyn_cast<MDNode>(MD->operands().begin()->get()))
+    return;
+
+#ifndef NDEBUG
+  unsigned LastIdx = 0;
+#endif
+
+  unsigned InIdx = 0;
+  unsigned LastExplicitArgOffset =
+      MF.getSubtarget<GCNSubtarget>().getExplicitKernelArgOffset();
+  for (auto &N : MD->operands()) {
+    auto *ArgNode = cast<MDNode>(N.get());
+    assert(ArgNode && ArgNode->getNumOperands() == 2);
+    unsigned ArgIdx =
+        mdconst::extract<ConstantInt>(ArgNode->getOperand(0))->getZExtValue();
+
+    while (InIdx < Ins.size() &&
+           (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
+      InIdx++;
+
+    for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
+           Ins[InIdx].getOrigArgIndex() == ArgIdx;
+         InIdx++) {
+#ifndef NDEBUG
+      // Verify sequential.
+      if (LastIdx != 0)
+        assert(LastIdx + 1 == InIdx);
+      LastIdx = InIdx;
+#endif
+
+      assert(ArgLocs[ArgIdx].isMemLoc());
+      auto &ArgLoc = ArgLocs[InIdx];
+      const Align KernelArgBaseAlign = Align(16);
+      unsigned ArgOffset = ArgLoc.getLocMemOffset();
+      Align Alignment = commonAlignment(KernelArgBaseAlign, ArgOffset);
+      unsigned NumAllocSGPRs =
+          alignTo(ArgLoc.getLocVT().getFixedSizeInBits(), 32) / 32;
+
+      // Arg is preloaded into the previous SGPR.
+      if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
+        Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
+            Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
+        continue;
+      }
+
+      unsigned Padding = ArgOffset - LastExplicitArgOffset;
+      const TargetRegisterClass *RC =
+          TRI.getSGPRClassForBitWidth(NumAllocSGPRs * 32);
+      SmallVectorImpl<MCRegister> *PreloadRegs =
+          Info.addPreloadedKernArg(TRI, RC, NumAllocSGPRs, InIdx, Padding);
+
+      if (PreloadRegs->size() > 1)
+        RC = &AMDGPU::SGPR_32RegClass;
+      for (auto &Reg : *PreloadRegs) {
+        assert(Reg);
+        MF.addLiveIn(Reg, RC);
+        CCInfo.AllocateReg(Reg);
+      }
+
+      LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
+    }
+  }
+}
+
+void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
+                                           const SIRegisterInfo &TRI,
+                                           SIMachineFunctionInfo &Info) const {
+  // Allways allocate this last since it is a synthetic preload.
   if (Info.hasLDSKernelId()) {
     Register Reg = Info.addLDSKernelId();
     MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
     CCInfo.AllocateReg(Reg);
   }
-
-  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
-  // these from the dispatch pointer.
 }
 
 // Allocate special input registers that are initialized per-wave.
 void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo,
                                            MachineFunction &MF,
                                            SIMachineFunctionInfo &Info,
                                            CallingConv::ID CallConv,
                                            bool IsShader) const {
if (Subtarget->isAmdPalOS()) {
         Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));
     }
   } else if (IsKernel) {
     assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
   } else {
     Splits.append(Ins.begin(), Ins.end());
   }
 
+  if (IsKernel)
+    analyzeFormalArgumentsCompute(CCInfo, Ins);
+
   if (IsEntryFunc) {
     allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
     allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
+    allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, *TRI, *Info);
+    allocateLDSKernelId(CCInfo, MF, *TRI, *Info);
   } else if (!IsGraphics) {
     // For the fixed ABI, pass workitem IDs in the last argument register.
     allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
   }
 
-  if (IsKernel) {
-    analyzeFormalArgumentsCompute(CCInfo, Ins);
-  } else {
+  if (!IsKernel) {
     CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
     CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
   }
 
   SmallVector<SDValue, 16> Chains;
 
   // FIXME: This is the minimum kernel argument alignment. We should improve
   // this to the maximum alignment of the arguments.
if (IsEntryFunc && VA.isMemLoc()) {
           Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
                                      Arg.Flags.getPointerAddrSpace());
         }
 
         InVals.push_back(Ptr);
         continue;
       }
 
-      SDValue Arg = lowerKernargMemParameter(
-        DAG, VT, MemVT, DL, Chain, Offset, Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
-      Chains.push_back(Arg.getValue(1));
+      SDValue NewArg;
+      if (Arg.isOrigArg() &&
+          Info->getArgInfo().PreloadKernArgs.count(Arg.getOrigArgIndex())) {
+        if (MemVT.getStoreSize() < 4 && Alignment < 4) {
+          // In this case the argument is packed into the previous preload SGPR.
+          int64_t AlignDownOffset = alignDown(Offset, 4);
+          int64_t OffsetDiff = Offset - AlignDownOffset;
+          EVT IntVT = MemVT.changeTypeToInteger();
+
+          const SIMachineFunctionInfo *Info =
+              MF.getInfo<SIMachineFunctionInfo>();
+          MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+          Register Reg = Info->getArgInfo()
+                             .PreloadKernArgs.find(ArgIdx - 1)
+                             ->getSecond()
+                             .Regs[0];
+
+          assert(Reg);
+          Register VReg = MRI.getLiveInVirtReg(Reg);
+          SDValue Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
+
+          SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, DL, MVT::i32);
+          SDValue Extract = DAG.getNode(ISD::SRL, DL, MVT::i32, Copy, ShiftAmt);
+
+          SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, Extract);
+          ArgVal = DAG.getNode(ISD::BITCAST, DL, MemVT, ArgVal);
+          NewArg = convertArgType(DAG, VT, MemVT, DL, ArgVal,
+                                  Ins[i].Flags.isSExt(), &Ins[i]);
+
+          NewArg = DAG.getMergeValues({NewArg, Copy.getValue(1)}, DL);
+        } else {
+          const SIMachineFunctionInfo *Info =
+              MF.getInfo<SIMachineFunctionInfo>();
+          MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+          const SmallVectorImpl<MCRegister> &PreloadRegs =
+              Info->getArgInfo()
+                  .PreloadKernArgs.find(ArgIdx - 1)
+                  ->getSecond()
+                  .Regs;
+
+          SDValue Copy;
+          if (PreloadRegs.size() == 1) {
+            Register VReg = MRI.getLiveInVirtReg(PreloadRegs[0]);
+            const TargetRegisterClass *RC = MRI.getRegClass(VReg);
+            NewArg = DAG.getCopyFromReg(
+                Chain, DL, VReg,
+                EVT::getIntegerVT(*DAG.getContext(),
+                                  TRI->getRegSizeInBits(*RC)));
+
+          } else {
+            // If the kernarg alignment does not match the alignment of the SGPR
+            // tuple RC that can accommodate this argument, it will be built up
+            // via copies from from the individual SGPRs that the argument was
+            // preloaded to.
+            SmallVector<SDValue, 4> Elts;
+            for (auto Reg : PreloadRegs) {
+              Register VReg = MRI.getLiveInVirtReg(Reg);
+              Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
+              Elts.push_back(Copy);
+            }
+            NewArg =
+                DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i32,
+                                                    PreloadRegs.size()),
+                                   DL, Elts);
+          }
+
+          SDValue CMemVT;
+          if (VT.isScalarInteger() && VT.bitsLT(NewArg.getSimpleValueType()))
+            CMemVT = DAG.getNode(ISD::TRUNCATE, DL, MemVT, NewArg);
+          else
+            CMemVT = DAG.getBitcast(MemVT, NewArg);
+          NewArg = convertArgType(DAG, VT, MemVT, DL, CMemVT,
+                                  Ins[i].Flags.isSExt(), &Ins[i]);
+          NewArg = DAG.getMergeValues({NewArg, Chain}, DL);
+        }
+      } else {
+        NewArg =
+            lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset,
+                                     Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
+      }
+      Chains.push_back(NewArg.getValue(1));
 
       auto *ParamTy =
           dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
       if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
-          ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
-                      ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
+          ParamTy &&
+          (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
         // On SI local pointers are just offsets into LDS, so they are always
         // less than 16-bits.  On CI and newer they could potentially be
         // real pointers, so we can't guarantee their size.
-        Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg,
+        NewArg = DAG.getNode(ISD::AssertZext, DL, NewArg.getValueType(), NewArg,
                              DAG.getValueType(MVT::i16));
       }
 
-      InVals.push_back(Arg);
+      InVals.push_back(NewArg);
       continue;
     } else if (!IsEntryFunc && VA.isMemLoc()) {
       SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
       InVals.push_back(Val);
       if (!Arg.Flags.isByVal())
         Chains.push_back(Val.getValue(1));
       continue;
     }

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h

 #include "AMDGPUArgumentUsageInfo.h"
 #include "AMDGPUMachineFunction.h"
 #include "AMDGPUTargetMachine.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIInstrInfo.h"
 #include "SIModeRegisterDefaults.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MIRYamlMapping.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
 namespace llvm {
 
 class MachineFrameInfo;
class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
   // unit. Minimum - first, maximum - second.
   std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
 
   const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV;
 
 private:
   unsigned NumUserSGPRs = 0;
   unsigned NumSystemSGPRs = 0;
+  unsigned NumKernargPreloadedSGPRs = 0;
 
   bool HasSpilledSGPRs = false;
   bool HasSpilledVGPRs = false;
   bool HasNonSpillStackObjects = false;
   bool IsStackRealigned = false;
 
   unsigned NumSpilledSGPRs = 0;
   unsigned NumSpilledVGPRs = 0;
public:
   Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
   Register addDispatchPtr(const SIRegisterInfo &TRI);
   Register addQueuePtr(const SIRegisterInfo &TRI);
   Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
   Register addDispatchID(const SIRegisterInfo &TRI);
   Register addFlatScratchInit(const SIRegisterInfo &TRI);
   Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
   Register addLDSKernelId();
+  SmallVectorImpl<MCRegister> *
+  addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
+                      unsigned AllocSizeDWord, int KernArgIdx, int Padding);
 
   /// Increment user SGPRs used for padding the argument list only.
   Register addReservedUserSGPR() {
     Register Next = getNextUserSGPR();
     ++NumUserSGPRs;
     return Next;
   }
 
public:
   unsigned getNumUserSGPRs() const {
     return NumUserSGPRs;
   }
 
   unsigned getNumPreloadedSGPRs() const {
     return NumUserSGPRs + NumSystemSGPRs;
   }
 
+  unsigned getNumKernargPreloadedSGPRs() const {
+    return NumKernargPreloadedSGPRs;
+  }
+
   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
   }
 
   /// Returns the physical register reserved for use as the resource
   /// descriptor for scratch accesses.
   Register getScratchRSrcReg() const {
     return ScratchRSrcReg;

llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp

 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "SIMachineFunctionInfo.h"
-#include "AMDGPUTargetMachine.h"
 #include "AMDGPUSubtarget.h"
+#include "AMDGPUTargetMachine.h"
 #include "GCNSubtarget.h"
-#include "SIRegisterInfo.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIRegisterInfo.h"
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MIRParser/MIParser.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/MIRParser/MIParser.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Function.h"
+#include "llvm/MC/MCRegister.h"
+#include "llvm/Support/MathExtras.h"
 #include <cassert>
 #include <optional>
 #include <vector>
 
 #define MAX_LANES 64
 
 using namespace llvm;
 

 }
 
 Register SIMachineFunctionInfo::addLDSKernelId() {
   ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());
   NumUserSGPRs += 1;
   return ArgInfo.LDSKernelId.getRegister();
 }
 
+SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(
+    const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
+    unsigned AllocSizeDWord, int KernArgIdx, int Padding) {
+  assert(!ArgInfo.PreloadKernArgs.count(KernArgIdx) &&
+         "Preload kernel argument allocated twice.");
+
+  unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
+  NumUserSGPRs += PaddingSGPRs;
+  // If the available register tuples are aligned with the kernarg to be
+  // preloaded use that register, otherwise we need to use a set of SGPRs and
+  // merge them.
+  if (Register PreloadReg =
+          TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC)) {
+    ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(PreloadReg);
+    NumUserSGPRs += AllocSizeDWord;
+  } else {
+    for (unsigned I = 0; I < AllocSizeDWord; ++I) {
+      ArgInfo.PreloadKernArgs[KernArgIdx].Regs.push_back(getNextUserSGPR());
+      NumUserSGPRs++;
+    }
+  }
+
+  // Track the actual number of SGPRs that HW will preload to.
+  NumKernargPreloadedSGPRs += AllocSizeDWord + PaddingSGPRs;
+  return &ArgInfo.PreloadKernArgs[KernArgIdx].Regs;
+}
+
 void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
                                              uint64_t Size, Align Alignment) {
   // Skip if it is an entry function or the register is already added.
   if (isEntryFunction() || WWMSpills.count(VGPR))
     return;
 
   WWMSpills.insert(std::make_pair(
       VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
auto convertArg = [&](std::optional<yaml::SIArgument> &A,
     // Check and update the optional mask.
     if (Arg.isMasked())
       SA.Mask = Arg.getMask();
 
     A = SA;
     return true;
   };
 
+  // TODO: Need to serialize kernarg preloads.
   bool Any = false;
   Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
   Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
   Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
   Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
   Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
   Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
   Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);

llvm/test/CodeGen/AMDGPU/gfx11-user-sgpr-init16-bug.ll

 
 ; GCN: .amdhsa_kernel minimal_kernel_inputs
 ; WORKAROUND: .amdhsa_user_sgpr_count 15
 ; NOWORKAROUND: .amdhsa_user_sgpr_count 0
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; GCN-NEXT: .amdhsa_wavefront_size32
 ; GCN-NEXT: .amdhsa_enable_private_segment 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
 ; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0

 
 ; GCN: .amdhsa_kernel minimal_kernel_inputs
 ; WORKAROUND: .amdhsa_user_sgpr_count 15
 ; NOWORKAROUND: .amdhsa_user_sgpr_count 0
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; GCN-NEXT: .amdhsa_wavefront_size32
 ; GCN-NEXT: .amdhsa_enable_private_segment 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
 ; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0

 
 ; GCN: .amdhsa_kernel queue_ptr
 ; WORKAROUND: .amdhsa_user_sgpr_count 15
 ; NOWORKAROUND: .amdhsa_user_sgpr_count 2
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 ; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; GCN-NEXT: .amdhsa_wavefront_size32
 ; GCN-NEXT: .amdhsa_enable_private_segment 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
 ; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0

 
 ; GCN: .amdhsa_kernel all_inputs
 ; WORKAROUND: .amdhsa_user_sgpr_count 13
 ; NOWORKAROUND: .amdhsa_user_sgpr_count 8
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 ; GCN-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 ; GCN-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 ; GCN-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; GCN-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GCN-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; GCN-NEXT: .amdhsa_wavefront_size32
 ; GCN-NEXT: .amdhsa_enable_private_segment 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 ; GCN-NEXT: .amdhsa_system_sgpr_workgroup_info 0
 ; GCN-NEXT: .amdhsa_system_vgpr_workitem_id 0

llvm/test/CodeGen/AMDGPU/preload-kernal-args-metadata.ll

 ; NO-PRELOAD-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !0
 ; NO-PRELOAD-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_2
 ; PRELOAD-1-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] !preload_kernel_args !0 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !2
-; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
+; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_2
 ; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[OUT:%.*]]) #[[ATTR0:[0-9]+]] !preload_kernel_args !0 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_2
 ; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[OUT:%.*]]) #[[ATTR0:[0-9]+]] !preload_kernel_args !0 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_2_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %load = load i32, ptr addrspace(1) %in
   store i32 %load, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @test_preload_metadata_kernel_4(ptr addrspace(1) %in, ptr addrspace(1) %in1, ptr addrspace(1) %out, ptr addrspace(1) %out1) #0 {

 ; NO-PRELOAD-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4
 ; PRELOAD-1-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) [[IN1:%.*]], ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !0 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 16
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 24
 ; PRELOAD-1-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !2
-; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
+; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4
-; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !4 {
+; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 16
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
 ; PRELOAD-3-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 24
-; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !5
+; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4
-; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !4 {
+; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 16
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_KERNARG_SEGMENT]], i64 24
-; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
+; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %load = load i32, ptr addrspace(1) %in
   %load1 = load i32, ptr addrspace(1) %in1
   store i32 %load, ptr addrspace(1) %out
   store i32 %load1, ptr addrspace(1) %out1
   ret void
 }

 ; NO-PRELOAD-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD2]], ptr addrspace(1) [[OUT2_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD3]], ptr addrspace(1) [[OUT3_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_8
 ; PRELOAD-1-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) [[IN1:%.*]], ptr addrspace(1) [[IN2:%.*]], ptr addrspace(1) [[IN3:%.*]], ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]], ptr addrspace(1) [[OUT2:%.*]], ptr addrspace(1) [[OUT3:%.*]]) #[[ATTR0]] !preload_kernel_args !0 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN2_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 16
 ; PRELOAD-1-NEXT:    [[IN2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN2_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN3_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 24
 ; PRELOAD-1-NEXT:    [[IN3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN3_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 32
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 40
 ; PRELOAD-1-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT2_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 48
 ; PRELOAD-1-NEXT:    [[OUT2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT2_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT3_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 56
 ; PRELOAD-1-NEXT:    [[OUT3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT3_KERNARG_OFFSET]], align 8, !invariant.load !2
-; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
+; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD2:%.*]] = load i32, ptr addrspace(1) [[IN2_LOAD]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD3:%.*]] = load i32, ptr addrspace(1) [[IN3_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD2]], ptr addrspace(1) [[OUT2_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD3]], ptr addrspace(1) [[OUT3_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_8
-; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[IN2:%.*]], ptr addrspace(1) [[IN3:%.*]], ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]], ptr addrspace(1) [[OUT2:%.*]], ptr addrspace(1) [[OUT3:%.*]]) #[[ATTR0]] !preload_kernel_args !4 {
+; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[IN2:%.*]], ptr addrspace(1) [[IN3:%.*]], ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]], ptr addrspace(1) [[OUT2:%.*]], ptr addrspace(1) [[OUT3:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[IN2_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 16
-; PRELOAD-3-NEXT:    [[IN2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN2_KERNARG_OFFSET]], align 16, !invariant.load !3
 ; PRELOAD-3-NEXT:    [[IN3_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 24
-; PRELOAD-3-NEXT:    [[IN3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN3_KERNARG_OFFSET]], align 8, !invariant.load !3
+; PRELOAD-3-NEXT:    [[IN3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN3_KERNARG_OFFSET]], align 8, !invariant.load !5
 ; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 32
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
+; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !5
 ; PRELOAD-3-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 40
-; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
+; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !5
 ; PRELOAD-3-NEXT:    [[OUT2_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 48
-; PRELOAD-3-NEXT:    [[OUT2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT2_KERNARG_OFFSET]], align 16, !invariant.load !3
+; PRELOAD-3-NEXT:    [[OUT2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT2_KERNARG_OFFSET]], align 16, !invariant.load !5
 ; PRELOAD-3-NEXT:    [[OUT3_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 56
-; PRELOAD-3-NEXT:    [[OUT3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT3_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[LOAD2:%.*]] = load i32, ptr addrspace(1) [[IN2_LOAD]], align 4
+; PRELOAD-3-NEXT:    [[OUT3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT3_KERNARG_OFFSET]], align 8, !invariant.load !5
+; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-3-NEXT:    [[LOAD2:%.*]] = load i32, ptr addrspace(1) [[IN2]], align 4
 ; PRELOAD-3-NEXT:    [[LOAD3:%.*]] = load i32, ptr addrspace(1) [[IN3_LOAD]], align 4
 ; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-3-NEXT:    store i32 [[LOAD2]], ptr addrspace(1) [[OUT2_LOAD]], align 4
 ; PRELOAD-3-NEXT:    store i32 [[LOAD3]], ptr addrspace(1) [[OUT3_LOAD]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_8
-; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[IN2:%.*]], ptr addrspace(1) inreg [[IN3:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]], ptr addrspace(1) inreg [[OUT2:%.*]], ptr addrspace(1) inreg [[OUT3:%.*]]) #[[ATTR0]] !preload_kernel_args !7 {
+; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[IN2:%.*]], ptr addrspace(1) inreg [[IN3:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]], ptr addrspace(1) inreg [[OUT2:%.*]], ptr addrspace(1) inreg [[OUT3:%.*]]) #[[ATTR0]] !preload_kernel_args !6 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(64) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN2_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 16
-; PRELOAD-8-NEXT:    [[IN2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN2_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN3_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 24
-; PRELOAD-8-NEXT:    [[IN3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN3_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 32
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
 ; PRELOAD-8-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 40
-; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
+; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !8
 ; PRELOAD-8-NEXT:    [[OUT2_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 48
-; PRELOAD-8-NEXT:    [[OUT2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT2_KERNARG_OFFSET]], align 16, !invariant.load !3
+; PRELOAD-8-NEXT:    [[OUT2_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT2_KERNARG_OFFSET]], align 16, !invariant.load !8
 ; PRELOAD-8-NEXT:    [[OUT3_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_8_KERNARG_SEGMENT]], i64 56
-; PRELOAD-8-NEXT:    [[OUT3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT3_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD2:%.*]] = load i32, ptr addrspace(1) [[IN2_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD3:%.*]] = load i32, ptr addrspace(1) [[IN3_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-8-NEXT:    [[OUT3_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT3_KERNARG_OFFSET]], align 8, !invariant.load !8
+; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-8-NEXT:    [[LOAD2:%.*]] = load i32, ptr addrspace(1) [[IN2]], align 4
+; PRELOAD-8-NEXT:    [[LOAD3:%.*]] = load i32, ptr addrspace(1) [[IN3]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-8-NEXT:    store i32 [[LOAD2]], ptr addrspace(1) [[OUT2_LOAD]], align 4
 ; PRELOAD-8-NEXT:    store i32 [[LOAD3]], ptr addrspace(1) [[OUT3_LOAD]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %load = load i32, ptr addrspace(1) %in
   %load1 = load i32, ptr addrspace(1) %in1
   %load2 = load i32, ptr addrspace(1) %in2

 ; NO-PRELOAD-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_inreg_offset
 ; PRELOAD-1-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !0 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 16
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 24
 ; PRELOAD-1-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !2
-; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
+; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_inreg_offset
 ; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !6 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 16
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 24
-; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
-; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
+; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
+; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_inreg_offset
-; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !4 {
+; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 16
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_KERNARG_SEGMENT]], i64 24
-; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
+; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %load = load i32, ptr addrspace(1) %in
   %load1 = load i32, ptr addrspace(1) %in1
   store i32 %load, ptr addrspace(1) %out
   store i32 %load1, ptr addrspace(1) %out1
   ret void
 }

 ; NO-PRELOAD-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_inreg_offset_two_sequence
 ; PRELOAD-1-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !0 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 16
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 24
 ; PRELOAD-1-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !2
-; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
+; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_inreg_offset_two_sequence
 ; PRELOAD-3-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !6 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 16
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 24
-; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
-; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
+; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-3-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
+; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_inreg_offset_two_sequence
-; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !4 {
+; PRELOAD-8-SAME: (ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(32) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 16
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_INREG_OFFSET_TWO_SEQUENCE_KERNARG_SEGMENT]], i64 24
-; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT_LOAD]], align 4
-; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
+; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD]], ptr addrspace(1) [[OUT]], align 4
+; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %load = load i32, ptr addrspace(1) %in
   %load1 = load i32, ptr addrspace(1) %in1
   store i32 %load, ptr addrspace(1) %out
   store i32 %load1, ptr addrspace(1) %out1
   ret void
 }

 ; NO-PRELOAD-NEXT:    [[ADD:%.*]] = add i32 [[LOAD]], [[EXT]]
 ; NO-PRELOAD-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_misaligned
 ; PRELOAD-1-SAME: (i16 inreg [[ARG0:%.*]], ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[IN1:%.*]], ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(40) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !2
-; PRELOAD-1-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
 ; PRELOAD-1-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 16
 ; PRELOAD-1-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 24
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 32
 ; PRELOAD-1-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 16, !invariant.load !2
 ; PRELOAD-1-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
 ; PRELOAD-1-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-1-NEXT:    [[EXT:%.*]] = zext i16 [[TMP2]] to i32
+; PRELOAD-1-NEXT:    [[EXT:%.*]] = zext i16 [[ARG0]] to i32
 ; PRELOAD-1-NEXT:    [[ADD:%.*]] = add i32 [[LOAD]], [[EXT]]
 ; PRELOAD-1-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_misaligned
 ; PRELOAD-3-SAME: (i16 inreg [[ARG0:%.*]], ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !8 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(40) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
-; PRELOAD-3-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 16
-; PRELOAD-3-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 16, !invariant.load !3
 ; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 24
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !3
+; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !5
 ; PRELOAD-3-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 32
-; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-3-NEXT:    [[EXT:%.*]] = zext i16 [[TMP2]] to i32
+; PRELOAD-3-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 16, !invariant.load !5
+; PRELOAD-3-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-3-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-3-NEXT:    [[EXT:%.*]] = zext i16 [[ARG0]] to i32
 ; PRELOAD-3-NEXT:    [[ADD:%.*]] = add i32 [[LOAD]], [[EXT]]
 ; PRELOAD-3-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-3-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_misaligned
 ; PRELOAD-8-SAME: (i16 inreg [[ARG0:%.*]], ptr addrspace(1) inreg [[IN:%.*]], ptr addrspace(1) inreg [[IN1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]], ptr addrspace(1) inreg [[OUT1:%.*]]) #[[ATTR0]] !preload_kernel_args !9 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(40) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
-; PRELOAD-8-NEXT:    [[IN_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[IN_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[IN1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 16
-; PRELOAD-8-NEXT:    [[IN1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[IN1_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 24
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !3
 ; PRELOAD-8-NEXT:    [[OUT1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_MISALIGNED_KERNARG_SEGMENT]], i64 32
-; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1_LOAD]], align 4
-; PRELOAD-8-NEXT:    [[EXT:%.*]] = zext i16 [[TMP2]] to i32
+; PRELOAD-8-NEXT:    [[OUT1_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT1_KERNARG_OFFSET]], align 16, !invariant.load !8
+; PRELOAD-8-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(1) [[IN]], align 4
+; PRELOAD-8-NEXT:    [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[IN1]], align 4
+; PRELOAD-8-NEXT:    [[EXT:%.*]] = zext i16 [[ARG0]] to i32
 ; PRELOAD-8-NEXT:    [[ADD:%.*]] = add i32 [[LOAD]], [[EXT]]
-; PRELOAD-8-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-8-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-8-NEXT:    store i32 [[LOAD1]], ptr addrspace(1) [[OUT1_LOAD]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %load = load i32, ptr addrspace(1) %in
   %load1 = load i32, ptr addrspace(1) %in1
   %ext = zext i16 %arg0 to i32
   %add = add i32 %load, %ext
   store i32 %add, ptr addrspace(1) %out

 ; NO-PRELOAD-NEXT:    [[EXT1:%.*]] = zext i16 [[TMP5]] to i32
 ; NO-PRELOAD-NEXT:    [[ADD:%.*]] = add i32 [[EXT]], [[EXT1]]
 ; NO-PRELOAD-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; NO-PRELOAD-NEXT:    ret void
 ;
 ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_i16_i16
 ; PRELOAD-1-SAME: (i16 inreg [[ARG0:%.*]], i16 [[ARG1:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] !preload_kernel_args !3 {
 ; PRELOAD-1-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-1-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !2
-; PRELOAD-1-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
 ; PRELOAD-1-NEXT:    [[ARG1_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 0
-; PRELOAD-1-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[ARG1_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !2
-; PRELOAD-1-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 16
-; PRELOAD-1-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP4]] to i16
+; PRELOAD-1-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG1_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !2
+; PRELOAD-1-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
+; PRELOAD-1-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
 ; PRELOAD-1-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 8
 ; PRELOAD-1-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !2
-; PRELOAD-1-NEXT:    [[EXT:%.*]] = zext i16 [[TMP2]] to i32
-; PRELOAD-1-NEXT:    [[EXT1:%.*]] = zext i16 [[TMP5]] to i32
+; PRELOAD-1-NEXT:    [[EXT:%.*]] = zext i16 [[ARG0]] to i32
+; PRELOAD-1-NEXT:    [[EXT1:%.*]] = zext i16 [[TMP3]] to i32
 ; PRELOAD-1-NEXT:    [[ADD:%.*]] = add i32 [[EXT]], [[EXT1]]
 ; PRELOAD-1-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
 ; PRELOAD-1-NEXT:    ret void
 ;
 ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_i16_i16
 ; PRELOAD-3-SAME: (i16 inreg [[ARG0:%.*]], i16 inreg [[ARG1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]]) #[[ATTR0]] !preload_kernel_args !10 {
 ; PRELOAD-3-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-3-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
-; PRELOAD-3-NEXT:    [[ARG1_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 0
-; PRELOAD-3-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[ARG1_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !3
-; PRELOAD-3-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 16
-; PRELOAD-3-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP4]] to i16
-; PRELOAD-3-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 8
-; PRELOAD-3-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-3-NEXT:    [[EXT:%.*]] = zext i16 [[TMP2]] to i32
-; PRELOAD-3-NEXT:    [[EXT1:%.*]] = zext i16 [[TMP5]] to i32
+; PRELOAD-3-NEXT:    [[EXT:%.*]] = zext i16 [[ARG0]] to i32
+; PRELOAD-3-NEXT:    [[EXT1:%.*]] = zext i16 [[ARG1]] to i32
 ; PRELOAD-3-NEXT:    [[ADD:%.*]] = add i32 [[EXT]], [[EXT1]]
-; PRELOAD-3-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-3-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-3-NEXT:    ret void
 ;
 ; PRELOAD-8-LABEL: define {{[^@]+}}@test_preload_metadata_kernel_4_i16_i16
 ; PRELOAD-8-SAME: (i16 inreg [[ARG0:%.*]], i16 inreg [[ARG1:%.*]], ptr addrspace(1) inreg [[OUT:%.*]]) #[[ATTR0]] !preload_kernel_args !11 {
 ; PRELOAD-8-NEXT:    [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
-; PRELOAD-8-NEXT:    [[ARG0_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[ARG0_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i16
-; PRELOAD-8-NEXT:    [[ARG1_KERNARG_OFFSET_ALIGN_DOWN:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 0
-; PRELOAD-8-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(4) [[ARG1_KERNARG_OFFSET_ALIGN_DOWN]], align 16, !invariant.load !3
-; PRELOAD-8-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP3]], 16
-; PRELOAD-8-NEXT:    [[TMP5:%.*]] = trunc i32 [[TMP4]] to i16
-; PRELOAD-8-NEXT:    [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TEST_PRELOAD_METADATA_KERNEL_4_I16_I16_KERNARG_SEGMENT]], i64 8
-; PRELOAD-8-NEXT:    [[OUT_LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 8, !invariant.load !3
-; PRELOAD-8-NEXT:    [[EXT:%.*]] = zext i16 [[TMP2]] to i32
-; PRELOAD-8-NEXT:    [[EXT1:%.*]] = zext i16 [[TMP5]] to i32
+; PRELOAD-8-NEXT:    [[EXT:%.*]] = zext i16 [[ARG0]] to i32
+; PRELOAD-8-NEXT:    [[EXT1:%.*]] = zext i16 [[ARG1]] to i32
 ; PRELOAD-8-NEXT:    [[ADD:%.*]] = add i32 [[EXT]], [[EXT1]]
-; PRELOAD-8-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT_LOAD]], align 4
+; PRELOAD-8-NEXT:    store i32 [[ADD]], ptr addrspace(1) [[OUT]], align 4
 ; PRELOAD-8-NEXT:    ret void
 ;
   %ext = zext i16 %arg0 to i32
   %ext1 = zext i16 %arg1 to i32
   %add = add i32 %ext, %ext1
   store i32 %add, ptr addrspace(1) %out
   ret void
 }
 
 attributes #0 = { nounwind }
-;.
 ; NO-PRELOAD: attributes #[[ATTR0]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; NO-PRELOAD: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
 ; PRELOAD-1: attributes #[[ATTR0]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; PRELOAD-1: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
 ; PRELOAD-3: attributes #[[ATTR0]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; PRELOAD-3: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
 ; PRELOAD-8: attributes #[[ATTR0]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" }
 ; PRELOAD-8: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
 ;.
 ; NO-PRELOAD: [[META0:![0-9]+]] = !{}
 ;.
 ; PRELOAD-1: [[META0:![0-9]+]] = !{!1}
 ; PRELOAD-1: [[META1:![0-9]+]] = !{i32 0, i32 2}
 ; PRELOAD-1: [[META2:![0-9]+]] = !{}
 ; PRELOAD-1: [[META3:![0-9]+]] = !{!4}
 ; PRELOAD-1: [[META4:![0-9]+]] = !{i32 0, i32 1}
 ;.
 ; PRELOAD-3: [[META0:![0-9]+]] = !{!1, !2}
 ; PRELOAD-3: [[META1:![0-9]+]] = !{i32 0, i32 2}
 ; PRELOAD-3: [[META2:![0-9]+]] = !{i32 1, i32 2}
-; PRELOAD-3: [[META3:![0-9]+]] = !{}
-; PRELOAD-3: [[META4:![0-9]+]] = !{!1, !2, !5}
-; PRELOAD-3: [[META5:![0-9]+]] = !{i32 2, i32 2}
-; PRELOAD-3: [[META6:![0-9]+]] = !{!1, !2, !5, !7}
+; PRELOAD-3: [[META3:![0-9]+]] = !{!1, !2, !4}
+; PRELOAD-3: [[META4:![0-9]+]] = !{i32 2, i32 2}
+; PRELOAD-3: [[META5:![0-9]+]] = !{}
+; PRELOAD-3: [[META6:![0-9]+]] = !{!1, !2, !4, !7}
 ; PRELOAD-3: [[META7:![0-9]+]] = !{i32 3, i32 2}
-; PRELOAD-3: [[META8:![0-9]+]] = !{!9, !2, !5}
+; PRELOAD-3: [[META8:![0-9]+]] = !{!9, !2, !4}
 ; PRELOAD-3: [[META9:![0-9]+]] = !{i32 0, i32 1}
-; PRELOAD-3: [[META10:![0-9]+]] = !{!9, !11, !5}
+; PRELOAD-3: [[META10:![0-9]+]] = !{!9, !11, !4}
 ; PRELOAD-3: [[META11:![0-9]+]] = !{i32 1, i32 1}
 ;.
 ; PRELOAD-8: [[META0:![0-9]+]] = !{!1, !2}
 ; PRELOAD-8: [[META1:![0-9]+]] = !{i32 0, i32 2}
 ; PRELOAD-8: [[META2:![0-9]+]] = !{i32 1, i32 2}
-; PRELOAD-8: [[META3:![0-9]+]] = !{}
-; PRELOAD-8: [[META4:![0-9]+]] = !{!1, !2, !5, !6}
-; PRELOAD-8: [[META5:![0-9]+]] = !{i32 2, i32 2}
-; PRELOAD-8: [[META6:![0-9]+]] = !{i32 3, i32 2}
-; PRELOAD-8: [[META7:![0-9]+]] = !{!1, !2, !5, !6, !8}
-; PRELOAD-8: [[META8:![0-9]+]] = !{i32 4, i32 2}
-; PRELOAD-8: [[META9:![0-9]+]] = !{!10, !2, !5, !6}
+; PRELOAD-8: [[META3:![0-9]+]] = !{!1, !2, !4, !5}
+; PRELOAD-8: [[META4:![0-9]+]] = !{i32 2, i32 2}
+; PRELOAD-8: [[META5:![0-9]+]] = !{i32 3, i32 2}
+; PRELOAD-8: [[META6:![0-9]+]] = !{!1, !2, !4, !5, !7}
+; PRELOAD-8: [[META7:![0-9]+]] = !{i32 4, i32 2}
+; PRELOAD-8: [[META8:![0-9]+]] = !{}
+; PRELOAD-8: [[META9:![0-9]+]] = !{!10, !2, !4, !5}
 ; PRELOAD-8: [[META10:![0-9]+]] = !{i32 0, i32 1}
-; PRELOAD-8: [[META11:![0-9]+]] = !{!10, !12, !5}
+; PRELOAD-8: [[META11:![0-9]+]] = !{!10, !12, !4}
 ; PRELOAD-8: [[META12:![0-9]+]] = !{i32 1, i32 1}
 ;.

llvm/test/CodeGen/AMDGPU/preload-kernal-args.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=NO-PRELOAD %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-1 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-2 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-4 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=PRELOAD-8 %s
+
+define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) {
+; NO-PRELOAD-LABEL: ptr1_i8:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xff
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: ptr1_i8:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    s_and_b32 s0, s0, 0xff
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: ptr1_i8:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_and_b32 s0, s8, 0xff
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: ptr1_i8:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_and_b32 s0, s8, 0xff
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: ptr1_i8:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_and_b32 s0, s8, 0xff
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  %ext = zext i8 %arg0 to i32
+  store i32 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %arg0) {
+; NO-PRELOAD-LABEL: ptr1_i8_zext_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xff
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: ptr1_i8_zext_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    s_and_b32 s0, s0, 0xff
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: ptr1_i8_zext_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_mov_b32 s0, 0xffff
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: ptr1_i8_zext_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_mov_b32 s0, 0xffff
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: ptr1_i8_zext_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_mov_b32 s0, 0xffff
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  %ext = zext i8 %arg0 to i32
+  store i32 %ext, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0) {
+; NO-PRELOAD-LABEL: ptr1_i16_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xffff
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: ptr1_i16_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    s_and_b32 s0, s0, 0xffff
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: ptr1_i16_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_and_b32 s0, s8, 0xffff
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: ptr1_i16_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_and_b32 s0, s8, 0xffff
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: ptr1_i16_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_and_b32 s0, s8, 0xffff
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  %ext = zext i16 %arg0 to i32
+  store i32 %ext, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0) {
+; NO-PRELOAD-LABEL: ptr1_i32_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: ptr1_i32_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: ptr1_i32_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: ptr1_i32_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: ptr1_i32_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store i32 %arg0, ptr addrspace(1) %out
+  ret void
+}
+
+; Check alignment on the second preloaded arg.
+
+define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) %out, i32 %arg1) {
+; NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x10
+; NO-PRELOAD-NEXT:    s_load_dword s3, s[4:5], 0x0
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    s_add_i32 s2, s3, s2
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x10
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    s_add_i32 s2, s6, s2
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_load_dword s0, s[4:5], 0x10
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-2-NEXT:    s_add_i32 s0, s6, s0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[8:9]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_add_i32 s0, s6, s10
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[8:9]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_add_i32 s0, s6, s10
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[8:9]
+; PRELOAD-8-NEXT:    s_endpgm
+  %add = add i32 %arg0, %arg1
+  store i32 %add, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0, i16 %arg1) {
+; NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    s_lshr_b32 s3, s2, 16
+; NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xffff
+; NO-PRELOAD-NEXT:    s_add_i32 s2, s2, s3
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    s_lshr_b32 s1, s0, 16
+; PRELOAD-1-NEXT:    s_and_b32 s0, s0, 0xffff
+; PRELOAD-1-NEXT:    s_add_i32 s0, s0, s1
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-2-NEXT:    s_and_b32 s1, s8, 0xffff
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s0, 16
+; PRELOAD-2-NEXT:    s_add_i32 s0, s1, s0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-4-NEXT:    s_and_b32 s1, s8, 0xffff
+; PRELOAD-4-NEXT:    s_add_i32 s0, s1, s0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-8-NEXT:    s_and_b32 s1, s8, 0xffff
+; PRELOAD-8-NEXT:    s_add_i32 s0, s1, s0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  %ext = zext i16 %arg0 to i32
+  %ext1 = zext i16 %arg1 to i32
+  %add = add i32 %ext, %ext1
+  store i32 %add, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> %in) {
+; NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
+; NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: ptr1_v2i8_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dword s0, s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    global_store_short v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: ptr1_v2i8_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, 0
+; PRELOAD-2-NEXT:    global_store_short v1, v0, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: ptr1_v2i8_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, 0
+; PRELOAD-4-NEXT:    global_store_short v1, v0, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: ptr1_v2i8_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, 0
+; PRELOAD-8-NEXT:    global_store_short v1, v0, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <2 x i8> %in, ptr addrspace(1) %out
+  ret void
+}
+
+; Don't try to preload byref args.
+
+define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 %after.offset) {
+; NO-PRELOAD-LABEL: byref_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s1
+; NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3]
+; NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
+; NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[2:3]
+; NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: byref_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s1
+; PRELOAD-1-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-1-NEXT:    global_store_dword v0, v2, s[6:7]
+; PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: byref_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s1
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-2-NEXT:    global_store_dword v0, v2, s[6:7]
+; PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: byref_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s1
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-4-NEXT:    global_store_dword v0, v2, s[6:7]
+; PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: byref_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s1
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-8-NEXT:    global_store_dword v0, v2, s[6:7]
+; PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
+; PRELOAD-8-NEXT:    s_endpgm
+  %in = load i32, ptr addrspace(4) %in.byref
+  store volatile i32 %in, ptr addrspace(1) %out, align 4
+  store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; TODO: Should do partial preload in cases like these where only part of the arg
+; can be preloaded.
+
+define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> %in) nounwind {
+; NO-PRELOAD-LABEL: v8i32_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s12
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s13
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s14
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s15
+; NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; NO-PRELOAD-NEXT:    s_nop 0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
+; NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v8i32_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-1-NEXT:    s_nop 0
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v8i32_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-2-NEXT:    s_nop 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v8i32_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-4-NEXT:    s_nop 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v8i32_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-8-NEXT:    s_nop 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <8 x i32> %in, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 x i16> %in) nounwind {
+; NO-PRELOAD-LABEL: v3i16_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
+; NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1] offset:4
+; NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v3i16_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s0
+; PRELOAD-1-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; PRELOAD-1-NEXT:    global_store_dword v0, v2, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v3i16_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-2-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-2-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v3i16_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-4-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-4-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v3i16_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-8-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s8
+; PRELOAD-8-NEXT:    global_store_dword v0, v1, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <3 x i16> %in, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 x i32> %in) nounwind {
+; NO-PRELOAD-LABEL: v3i32_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s1
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
+; NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v3i32_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s0
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
+; PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v3i32_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s10
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s11
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s12
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v3i32_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s10
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s11
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s12
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v3i32_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s10
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s11
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s12
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <3 x i32> %in, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 x float> %in) nounwind {
+; NO-PRELOAD-LABEL: v3f32_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s1
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
+; NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v3f32_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s0
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
+; PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v3f32_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s10
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s11
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s12
+; PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v3f32_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s10
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s11
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s12
+; PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v3f32_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s10
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s11
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s12
+; PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <3 x float> %in, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 x i8> %in) nounwind {
+; NO-PRELOAD-LABEL: v5i8_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
+; NO-PRELOAD-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
+; NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v5i8_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s0
+; PRELOAD-1-NEXT:    global_store_byte v0, v1, s[6:7] offset:4
+; PRELOAD-1-NEXT:    global_store_dword v0, v2, s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v5i8_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 24
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s9
+; PRELOAD-2-NEXT:    global_store_byte v1, v2, s[6:7] offset:4
+; PRELOAD-2-NEXT:    global_store_dword v1, v0, s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v5i8_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 24
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s9
+; PRELOAD-4-NEXT:    global_store_byte v1, v2, s[6:7] offset:4
+; PRELOAD-4-NEXT:    global_store_dword v1, v0, s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v5i8_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 24
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s9
+; PRELOAD-8-NEXT:    global_store_byte v1, v2, s[6:7] offset:4
+; PRELOAD-8-NEXT:    global_store_dword v1, v0, s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <5 x i8> %in, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x double> %in) nounwind {
+; NO-PRELOAD-LABEL: v5f64_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; NO-PRELOAD-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s12
+; NO-PRELOAD-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s13
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s14
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s15
+; NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
+; NO-PRELOAD-NEXT:    s_nop 0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
+; NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v5f64_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; PRELOAD-1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-1-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-1-NEXT:    s_nop 0
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v5f64_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; PRELOAD-2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-2-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-2-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-2-NEXT:    s_nop 0
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v5f64_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; PRELOAD-4-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-4-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-4-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-4-NEXT:    s_nop 0
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v5f64_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; PRELOAD-8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
+; PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-8-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s12
+; PRELOAD-8-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s13
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s14
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s15
+; PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; PRELOAD-8-NEXT:    s_nop 0
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
+; PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <5 x double> %in, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) {
+; NO-PRELOAD-LABEL: v8i8_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: v8i8_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: v8i8_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s9, 8
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s9, 24
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s9, 16
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 24
+; PRELOAD-2-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; PRELOAD-2-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: v8i8_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s9, 8
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s9, 24
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s9, 16
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 24
+; PRELOAD-4-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; PRELOAD-4-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: v8i8_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s9, 8
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s9, 24
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s9, 16
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 8
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 24
+; PRELOAD-8-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; PRELOAD-8-NEXT:    s_lshr_b32 s0, s8, 16
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store <8 x i8> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) {
+; NO-PRELOAD-LABEL: i64_kernel_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
+; NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: i64_kernel_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: i64_kernel_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-2-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: i64_kernel_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-4-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: i64_kernel_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-8-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store i64 %a, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double %in) {
+; NO-PRELOAD-LABEL: f64_kernel_preload_arg:
+; NO-PRELOAD:       ; %bb.0:
+; NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
+; NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
+; NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
+; NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; NO-PRELOAD-NEXT:    s_endpgm
+;
+; PRELOAD-1-LABEL: f64_kernel_preload_arg:
+; PRELOAD-1:       ; %bb.0:
+; PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
+; PRELOAD-1-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-1-NEXT:    s_endpgm
+;
+; PRELOAD-2-LABEL: f64_kernel_preload_arg:
+; PRELOAD-2:       ; %bb.0:
+; PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-2-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-2-NEXT:    s_endpgm
+;
+; PRELOAD-4-LABEL: f64_kernel_preload_arg:
+; PRELOAD-4:       ; %bb.0:
+; PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-4-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-4-NEXT:    s_endpgm
+;
+; PRELOAD-8-LABEL: f64_kernel_preload_arg:
+; PRELOAD-8:       ; %bb.0:
+; PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
+; PRELOAD-8-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; PRELOAD-8-NEXT:    s_endpgm
+  store double %in, ptr addrspace(1) %out
+  ret void
+}

llvm/test/CodeGen/AMDGPU/stack-realign-kernel.ll

 ; VI-NEXT:     .amdhsa_kernarg_size 0
 ; VI-NEXT:     .amdhsa_user_sgpr_count 6
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_id 0
 ; VI-NEXT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; VI-NEXT:     .amdhsa_user_sgpr_kernarg_preload_length  0
+; VI-NEXT:     .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_size 0
 ; VI-NEXT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_x 1
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_y 0
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_z 0
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_info 0
 ; VI-NEXT:     .amdhsa_system_vgpr_workitem_id 0
 ; VI-NEXT:     .amdhsa_next_free_vgpr 1

 ; GFX9-NEXT:     .amdhsa_kernarg_size 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_count 6
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_id 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_preload_length  0
+; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_size 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_x 1
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_y 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_z 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_info 0
 ; GFX9-NEXT:     .amdhsa_system_vgpr_workitem_id 0
 ; GFX9-NEXT:     .amdhsa_next_free_vgpr 1

 ; VI-NEXT:     .amdhsa_kernarg_size 0
 ; VI-NEXT:     .amdhsa_user_sgpr_count 6
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_id 0
 ; VI-NEXT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; VI-NEXT:     .amdhsa_user_sgpr_kernarg_preload_length  0
+; VI-NEXT:     .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_size 0
 ; VI-NEXT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_x 1
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_y 0
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_z 0
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_info 0
 ; VI-NEXT:     .amdhsa_system_vgpr_workitem_id 0
 ; VI-NEXT:     .amdhsa_next_free_vgpr 1

 ; GFX9-NEXT:     .amdhsa_kernarg_size 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_count 6
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_id 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_preload_length  0
+; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_size 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_x 1
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_y 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_z 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_info 0
 ; GFX9-NEXT:     .amdhsa_system_vgpr_workitem_id 0
 ; GFX9-NEXT:     .amdhsa_next_free_vgpr 1

 ; VI-NEXT:     .amdhsa_kernarg_size 0
 ; VI-NEXT:     .amdhsa_user_sgpr_count 6
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; VI-NEXT:     .amdhsa_user_sgpr_dispatch_id 0
 ; VI-NEXT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; VI-NEXT:     .amdhsa_user_sgpr_kernarg_preload_length  0
+; VI-NEXT:     .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; VI-NEXT:     .amdhsa_user_sgpr_private_segment_size 0
 ; VI-NEXT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_x 1
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_y 0
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_id_z 0
 ; VI-NEXT:     .amdhsa_system_sgpr_workgroup_info 0
 ; VI-NEXT:     .amdhsa_system_vgpr_workitem_id 0
 ; VI-NEXT:     .amdhsa_next_free_vgpr 1

 ; GFX9-NEXT:     .amdhsa_kernarg_size 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_count 6
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_buffer 1
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_queue_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_dispatch_id 0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_flat_scratch_init 1
+; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_preload_length  0
+; GFX9-NEXT:     .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; GFX9-NEXT:     .amdhsa_user_sgpr_private_segment_size 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_x 1
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_y 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_id_z 0
 ; GFX9-NEXT:     .amdhsa_system_sgpr_workgroup_info 0
 ; GFX9-NEXT:     .amdhsa_system_vgpr_workitem_id 0
 ; GFX9-NEXT:     .amdhsa_next_free_vgpr 1

llvm/test/MC/AMDGPU/hsa-gfx10-v3.s

 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=3 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 --amdhsa-code-object-version=3 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        0000c0 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 // READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
 // READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 // READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac60 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0070 015001e4 1f0f007f 7f040000 00000000
+// OBJDUMP-NEXT: 0070 015001e4 1f0f007f 7f048100 00000000
 // special_sgpr
 // OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00b0 00000060 80000000 00000000 00000000
 
 .text
 // ASM: .text
.amdhsa_kernel complete
   .amdhsa_private_segment_fixed_size 1
   .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
   .amdhsa_user_sgpr_flat_scratch_init 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_wavefront_size32 1
   .amdhsa_system_sgpr_private_segment_wavefront_offset 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1

 // ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_count 15
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
 // ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_wavefront_size32 1
 // ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1

llvm/test/MC/AMDGPU/hsa-gfx11-v3.s

 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 --amdhsa-code-object-version=3 < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 --amdhsa-code-object-version=3 -filetype=obj < %s > %t
 // RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        0000c0 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 // READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
 // READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 // READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac60 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0070 015001e4 130f007f 5e040000 00000000
+// OBJDUMP-NEXT: 0070 015001e4 130f007f 5e048100 00000000
 // special_sgpr
 // OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00b0 00000060 80000000 00000000 00000000
 
 .text
 // ASM: .text

 .amdhsa_kernel complete
   .amdhsa_group_segment_fixed_size 1
   .amdhsa_private_segment_fixed_size 1
   .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_wavefront_size32 1
   .amdhsa_enable_private_segment 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1

 // ASM-NEXT: .amdhsa_group_segment_fixed_size 1
 // ASM-NEXT: .amdhsa_private_segment_fixed_size 1
 // ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_count 9
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_wavefront_size32 1
 // ASM-NEXT: .amdhsa_enable_private_segment 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1

llvm/test/MC/AMDGPU/hsa-gfx90a-v3.s

 // RUN: llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t
 // RUN: llvm-readobj --elf-output-style=GNU --sections --symbols --relocations %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000080 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 00000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100
-// OBJDUMP-NEXT: 0070 c1500104 1f0f007f 7f000000 00000000
+// OBJDUMP-NEXT: 0070 c1500104 1f0f007f 7f008100 00000000
 
 .text
 // ASM: .text
 
 .amdgcn_target "amdgcn-amd-amdhsa--gfx90a+xnack+sram-ecc"
 // ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx90a+xnack+sram-ecc"
 
 .p2align 8
.amdhsa_kernel complete
   .amdhsa_group_segment_fixed_size 1
   .amdhsa_private_segment_fixed_size 1
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
   .amdhsa_user_sgpr_flat_scratch_init 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_system_sgpr_private_segment_wavefront_offset 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1
   .amdhsa_next_free_vgpr 9

 // ASM-NEXT: .amdhsa_kernarg_size 0
 // ASM-NEXT: .amdhsa_user_sgpr_count 15
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
 // ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
 // ASM-NEXT: .amdhsa_next_free_vgpr 9

llvm/test/MC/AMDGPU/hsa-gfx940-v3.s

 // RUN: llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx940 < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc --amdhsa-code-object-version=3 -triple amdgcn-amd-amdhsa -mcpu=gfx940 -filetype=obj < %s > %t
 // RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000080 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-DAG: {{[0-9]+}}: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 00000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100
-// OBJDUMP-NEXT: 0070 01510104 130f007f 5e000000 00000000
+// OBJDUMP-NEXT: 0070 01510104 130f007f 5e008100 00000000
 
 .text
 // ASM: .text
 
 .amdgcn_target "amdgcn-amd-amdhsa--gfx940+xnack+sram-ecc"
 // ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx940+xnack+sram-ecc"
 
 .p2align 8

 .p2align 6
 .amdhsa_kernel complete
   .amdhsa_group_segment_fixed_size 1
   .amdhsa_private_segment_fixed_size 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_enable_private_segment 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1
   .amdhsa_next_free_vgpr 9

 // ASM-NEXT: .amdhsa_group_segment_fixed_size 1
 // ASM-NEXT: .amdhsa_private_segment_fixed_size 1
 // ASM-NEXT: .amdhsa_kernarg_size 0
 // ASM-NEXT: .amdhsa_user_sgpr_count 9
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_enable_private_segment 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
 // ASM-NEXT: .amdhsa_next_free_vgpr 9

llvm/test/MC/AMDGPU/hsa-v3.s

 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=3 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=3 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000100 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 // READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
 // READOBJ: 00000000000000d0 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 310
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
 // READOBJ-NEXT: 0000000000000300  0 FUNC    LOCAL  PROTECTED 2 disabled_user_sgpr
 // READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 // READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
 // READOBJ-NEXT: 00000000000000c0 64 OBJECT  LOCAL  DEFAULT   3 disabled_user_sgpr.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f000000 00000000
+// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f008100 00000000
 // special_sgpr
 // OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00b0 00010000 80000000 00000000 00000000
 // disabled_user_sgpr
 // OBJDUMP-NEXT: 00c0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00d0 00000000 00000000 00000000 00000000
.amdhsa_kernel complete
   .amdhsa_private_segment_fixed_size 1
   .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
   .amdhsa_user_sgpr_flat_scratch_init 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_system_sgpr_private_segment_wavefront_offset 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1
   .amdhsa_next_free_vgpr 9

 // ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_count 15
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
 // ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
 // ASM-NEXT: .amdhsa_next_free_vgpr 9

llvm/test/MC/AMDGPU/hsa-v4.s

 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=4 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=4 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000100 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 // READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
 // READOBJ: 00000000000000d0 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 310
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
 // READOBJ-NEXT: 0000000000000300  0 FUNC    LOCAL  PROTECTED 2 disabled_user_sgpr
 // READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 // READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
 // READOBJ-NEXT: 00000000000000c0 64 OBJECT  LOCAL  DEFAULT   3 disabled_user_sgpr.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f000000 00000000
+// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f008100 00000000
 // special_sgpr
 // OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00b0 00010000 80000000 00000000 00000000
 // disabled_user_sgpr
 // OBJDUMP-NEXT: 00c0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00d0 00000000 00000000 00000000 00000000
.amdhsa_kernel complete
   .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_count 15
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
   .amdhsa_user_sgpr_flat_scratch_init 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_system_sgpr_private_segment_wavefront_offset 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1
   .amdhsa_next_free_vgpr 9
   .amdhsa_next_free_sgpr 27
   .amdhsa_reserve_vcc 0
   .amdhsa_reserve_flat_scratch 0
   .amdhsa_reserve_xnack_mask 1
   .amdhsa_float_round_mode_32 1
   .amdhsa_float_round_mode_16_64 1
   .amdhsa_float_denorm_mode_32 1
   .amdhsa_float_denorm_mode_16_64 0

 // ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_count 15
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
 // ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
 // ASM-NEXT: .amdhsa_next_free_vgpr 9

llvm/test/MC/AMDGPU/hsa-v5-uses-dynamic-stack.s

 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=5 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
 // RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx904 --amdhsa-code-object-version=5 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
 // READOBJ: Section Headers
 // READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
 // READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000100 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
 
 // READOBJ: Relocation section '.rela.rodata' at offset
 // READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
-// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
 // READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
 // READOBJ: 00000000000000d0 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 310
 
 // READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
 // READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
 // READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
 // READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
 // READOBJ-NEXT: 0000000000000300  0 FUNC    LOCAL  PROTECTED 2 disabled_user_sgpr
 // READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
 // READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
 // READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
 // READOBJ-NEXT: 00000000000000c0 64 OBJECT  LOCAL  DEFAULT   3 disabled_user_sgpr.kd
 
 // OBJDUMP: Contents of section .rodata
 // Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
 // minimal
 // OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000
 // complete
 // OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
 // OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
-// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f080000 00000000
+// OBJDUMP-NEXT: 0070 c2500104 1f0f007f 7f088100 00000000
 // special_sgpr
 // OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00b0 00010000 80000000 00000000 00000000
 // disabled_user_sgpr
 // OBJDUMP-NEXT: 00c0 00000000 00000000 00000000 00000000
 // OBJDUMP-NEXT: 00d0 00000000 00000000 00000000 00000000
.amdhsa_kernel complete
   .amdhsa_kernarg_size 8
   .amdhsa_user_sgpr_count 15
   .amdhsa_user_sgpr_private_segment_buffer 1
   .amdhsa_user_sgpr_dispatch_ptr 1
   .amdhsa_user_sgpr_queue_ptr 1
   .amdhsa_user_sgpr_kernarg_segment_ptr 1
   .amdhsa_user_sgpr_dispatch_id 1
   .amdhsa_user_sgpr_flat_scratch_init 1
+  .amdhsa_user_sgpr_kernarg_preload_length  1
+  .amdhsa_user_sgpr_kernarg_preload_offset  1
   .amdhsa_user_sgpr_private_segment_size 1
   .amdhsa_uses_dynamic_stack 1
   .amdhsa_system_sgpr_private_segment_wavefront_offset 1
   .amdhsa_system_sgpr_workgroup_id_x 0
   .amdhsa_system_sgpr_workgroup_id_y 1
   .amdhsa_system_sgpr_workgroup_id_z 1
   .amdhsa_system_sgpr_workgroup_info 1
   .amdhsa_system_vgpr_workitem_id 1

 // ASM-NEXT: .amdhsa_kernarg_size 8
 // ASM-NEXT: .amdhsa_user_sgpr_count 15
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
 // ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
 // ASM-NEXT: .amdhsa_uses_dynamic_stack 1
 // ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
 // ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
 // ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1

llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx10.s

 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; CHECK-NEXT: .amdhsa_wavefront_size32 1
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_wavefront_size32 1
 .end_amdhsa_kernel
 
 ;--- 2.s

 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; CHECK-NEXT: .amdhsa_wavefront_size32 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_shared_vgpr_count 0
 .end_amdhsa_kernel
 
 ;--- 3.s

 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; CHECK-NEXT: .amdhsa_wavefront_size32 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_shared_vgpr_count 1
 .end_amdhsa_kernel
 
 ;--- 4.s

 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
 ; CHECK-NEXT: .amdhsa_wavefront_size32 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 32
   .amdhsa_shared_vgpr_count 1
   .amdhsa_wavefront_size32 0
 .end_amdhsa_kernel

llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx90a.s

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 0
   .amdhsa_next_free_sgpr 0
   .amdhsa_accum_offset 4
 .end_amdhsa_kernel
 
 ;--- 2.s

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 0
   .amdhsa_accum_offset 8
 .end_amdhsa_kernel
 
 ;--- 3.s

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  2
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 0
   .amdhsa_accum_offset 12
+  .amdhsa_user_sgpr_kernarg_preload_length 2
+  .amdhsa_user_sgpr_kernarg_preload_offset 1
 .end_amdhsa_kernel

llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-sgpr.s

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 0
   .amdhsa_next_free_sgpr 42
   .amdhsa_reserve_flat_scratch 0
   .amdhsa_reserve_xnack_mask 0
   .amdhsa_reserve_vcc 0
 .end_amdhsa_kernel

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 0
   .amdhsa_next_free_sgpr 0
   .amdhsa_reserve_flat_scratch 1
   .amdhsa_reserve_xnack_mask 0
   .amdhsa_reserve_vcc 1
 .end_amdhsa_kernel

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  2
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  1
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 0
   .amdhsa_next_free_sgpr 35
   .amdhsa_reserve_flat_scratch 1
   .amdhsa_reserve_xnack_mask 0
   .amdhsa_reserve_vcc 1
+  .amdhsa_user_sgpr_kernarg_preload_length 2
+  .amdhsa_user_sgpr_kernarg_preload_offset 1
 .end_amdhsa_kernel

llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-vgpr.s

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 23
   .amdhsa_next_free_sgpr 0
 .end_amdhsa_kernel
 
 ;--- 2.s
 ; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=-xnack -filetype=obj -mcpu=gfx908 < 2.s > 2.o

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 14
   .amdhsa_next_free_sgpr 0
 .end_amdhsa_kernel
 
 ;--- 3.s
 ; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -mattr=-xnack -filetype=obj -mcpu=gfx908 < 3.s > 3.o

 ; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
 ; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_length  0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; CHECK-NEXT: .end_amdhsa_kernel
 .amdhsa_kernel kernel
   .amdhsa_next_free_vgpr 32
   .amdhsa_next_free_sgpr 0
 .end_amdhsa_kernel

llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-zeroed-gfx10.s

 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_private_segment_buffer 0
 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_dispatch_ptr 0
 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_queue_ptr 0
 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_kernarg_segment_ptr 0
 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_dispatch_id 0
 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_flat_scratch_init 0
 ; OBJDUMP-NEXT:         .amdhsa_user_sgpr_private_segment_size 0
 ; OBJDUMP-NEXT:         .amdhsa_wavefront_size32 0
+; OBJDUMP-NEXT:         .amdhsa_user_sgpr_kernarg_preload_length  0
+; OBJDUMP-NEXT:         .amdhsa_user_sgpr_kernarg_preload_offset  0
 ; OBJDUMP-NEXT: .end_amdhsa_kernel
 
 .amdhsa_kernel my_kernel
   .amdhsa_group_segment_fixed_size 0
   .amdhsa_private_segment_fixed_size 0
   .amdhsa_next_free_vgpr 8
   .amdhsa_reserve_vcc 0
   .amdhsa_reserve_flat_scratch 0