Implementation of variable-length load/store intrinsics

[hussainjk]

Summary

Implemented intrinsics llvm.variable.length.load and llvm.variable.length.store,
as well as infrastructure for lowering these using new corresponding SDAG nodes.

These intrinsics represent loading and storing vectors with a number of contiguous active lanes given at runtime.
In effect, they specialize masked loads and stores to the case when the mask can instead be specified by a length.

Please see our associated RFC for technical design discussion.

llvm/include/llvm/Analysis/TargetTransformInfo.h

public:
   /// \return True is LSR should make efforts to create/preserve post-inc
   /// addressing mode expressions.
   bool shouldFavorPostInc() const;
 
   /// Return true if LSR should make efforts to generate indexed addressing
   /// modes that operate across loop iterations.
   bool shouldFavorBackedgeIndex(const Loop *L) const;
 
+  /// Return true if the target supports variable-length load.
+  bool isLegalVariableLengthLoad(Type *DataType, Align Alignment,
+                                 Type *LengthType) const;
+  /// Return true if the target supports variable-length store.
+  bool isLegalVariableLengthStore(Type *DataType, Align Alignment,
+                                  Type *LengthType) const;
+
   /// Return true if the target supports masked store.
   bool isLegalMaskedStore(Type *DataType, Align Alignment) const;
   /// Return true if the target supports masked load.
   bool isLegalMaskedLoad(Type *DataType, Align Alignment) const;
 
   /// Return true if the target supports nontemporal store.
   bool isLegalNTStore(Type *DataType, Align Alignment) const;
   /// Return true if the target supports nontemporal load.
virtual bool isLSRCostLess(TargetTransformInfo::LSRCost &C1,
                              TargetTransformInfo::LSRCost &C2) = 0;
   virtual bool isProfitableLSRChainElement(Instruction *I) = 0;
   virtual bool canMacroFuseCmp() = 0;
   virtual bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE,
                           LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC,
                           TargetLibraryInfo *LibInfo) = 0;
   virtual bool shouldFavorPostInc() const = 0;
   virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0;
+  virtual bool isLegalVariableLengthLoad(Type *DataType, Align Alignment,
+                                         Type *LengthType) = 0;
+  virtual bool isLegalVariableLengthStore(Type *DataType, Align Alignment,
+                                          Type *LengthType) = 0;
   virtual bool isLegalMaskedStore(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedScatter(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedCompressStore(Type *DataType) = 0;
   virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0;
bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
                   DominatorTree *DT, AssumptionCache *AC,
                   TargetLibraryInfo *LibInfo) override {
     return Impl.canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
   }
   bool shouldFavorPostInc() const override { return Impl.shouldFavorPostInc(); }
   bool shouldFavorBackedgeIndex(const Loop *L) const override {
     return Impl.shouldFavorBackedgeIndex(L);
   }
+  bool isLegalVariableLengthLoad(Type *DataType, Align Alignment,
+                                 Type *LengthType) override {
+    return Impl.isLegalVariableLengthLoad(DataType, Alignment, LengthType);
+  }
+  bool isLegalVariableLengthStore(Type *DataType, Align Alignment,
+                                  Type *LengthType) override {
+    return Impl.isLegalVariableLengthStore(DataType, Alignment, LengthType);
+  }
   bool isLegalMaskedStore(Type *DataType, Align Alignment) override {
     return Impl.isLegalMaskedStore(DataType, Alignment);
   }
   bool isLegalMaskedLoad(Type *DataType, Align Alignment) override {
     return Impl.isLegalMaskedLoad(DataType, Alignment);
   }
   bool isLegalNTStore(Type *DataType, Align Alignment) override {
     return Impl.isLegalNTStore(DataType, Alignment);

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI,
                   TargetLibraryInfo *LibInfo) {
     return false;
   }
 
   bool shouldFavorPostInc() const { return false; }
 
   bool shouldFavorBackedgeIndex(const Loop *L) const { return false; }
 
+  bool isLegalVariableLengthLoad(Type *DataType, Align Alignment,
+                                 Type *LengthType) {
+    return false;
+  }
+  bool isLegalVariableLengthStore(Type *DataType, Align Alignment,
+                                  Type *LengthType) {
+    return false;
+  }
+
   bool isLegalMaskedStore(Type *DataType, Align Alignment) { return false; }
 
   bool isLegalMaskedLoad(Type *DataType, Align Alignment) { return false; }
 
   bool isLegalNTStore(Type *DataType, Align Alignment) {
     // By default, assume nontemporal memory stores are available for stores
     // that are aligned and have a size that is a power of 2.
     unsigned DataSize = DL.getTypeStoreSize(DataType);

llvm/include/llvm/CodeGen/ISDOpcodes.h

   ATOMIC_LOAD_NAND,
   ATOMIC_LOAD_MIN,
   ATOMIC_LOAD_MAX,
   ATOMIC_LOAD_UMIN,
   ATOMIC_LOAD_UMAX,
   ATOMIC_LOAD_FADD,
   ATOMIC_LOAD_FSUB,
 
+  // Variable-length load and store - consecutive vector load and store
+  // operations with
+  // additional length operand that prevents memory accesses beyond the first
+  // consecutive
+  // "length" lanes.
+  //
+  // Val, OutChain = VARLEN_LOAD(BasePtr, Length, PassThru)
+  // OutChain = VARLEN_STORE(Value, BasePtr, Length)
+  VARLEN_LOAD,
+  VARLEN_STORE,
+
   // Masked load and store - consecutive vector load and store operations
   // with additional mask operand that prevents memory accesses to the
   // masked-off lanes.
   //
   // Val, OutChain = MLOAD(BasePtr, Mask, PassThru)
   // OutChain = MSTORE(Value, BasePtr, Mask)
   MLOAD,
   MSTORE,

llvm/include/llvm/CodeGen/Passes.h

namespace llvm {
   /// If AbortOnFailedISel is true, abort compilation instead of resetting.
   MachineFunctionPass *createResetMachineFunctionPass(bool EmitFallbackDiag,
                                                       bool AbortOnFailedISel);
 
   /// createCodeGenPreparePass - Transform the code to expose more pattern
   /// matching during instruction selection.
   FunctionPass *createCodeGenPreparePass();
 
+  /// createScalarizeVariableLengthMemIntrinPass - Replace variable-length load
+  /// and store intrinsics with scalar code when target doesn't support them.
+  FunctionPass *createScalarizeVariableLengthMemIntrinPass();
+
   /// createScalarizeMaskedMemIntrinPass - Replace masked load, store, gather
   /// and scatter intrinsics with scalar code when target doesn't support them.
   FunctionPass *createScalarizeMaskedMemIntrinPass();
 
   /// AtomicExpandID -- Lowers atomic operations in terms of either cmpxchg
   /// load-linked/store-conditional loops.
   extern char &AtomicExpandID;
 

llvm/include/llvm/CodeGen/SelectionDAG.h

getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
     return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
                          MaybeAlign(Alignment), MMOFlags, AAInfo);
   }
   SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                         SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
   SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
                           SDValue Offset, ISD::MemIndexedMode AM);
 
+  SDValue getVariableLengthLoad(EVT VT, const SDLoc &dl, SDValue Chain,
+                                SDValue Base, SDValue Offset, SDValue Length,
+                                SDValue Src0, EVT MemVT, MachineMemOperand *MMO,
+                                ISD::MemIndexedMode AM, ISD::LoadExtType,
+                                bool IsExpanding = false);
+  SDValue getIndexedVariableLengthLoad(SDValue OrigLoad, const SDLoc &dl,
+                                       SDValue Base, SDValue Offset,
+                                       ISD::MemIndexedMode AM);
+  SDValue getVariableLengthStore(SDValue Chain, const SDLoc &dl, SDValue Val,
+                                 SDValue Base, SDValue Offset, SDValue Length,
+                                 EVT MemVT, MachineMemOperand *MMO,
+                                 ISD::MemIndexedMode AM,
+                                 bool IsTruncating = false);
+  SDValue getIndexedVariableLengthStore(SDValue OrigStore, const SDLoc &dl,
+                                        SDValue Base, SDValue Offset,
+                                        ISD::MemIndexedMode AM);
+
   SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base,
                         SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT,
                         MachineMemOperand *MMO, ISD::MemIndexedMode AM,
                         ISD::LoadExtType, bool IsExpanding = false);
   SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
                                SDValue Offset, ISD::MemIndexedMode AM);
   SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                          SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT,

llvm/include/llvm/CodeGen/SelectionDAGNodes.h

class MemSDNodeBitfields {
     uint16_t IsNonTemporal : 1;
     uint16_t IsDereferenceable : 1;
     uint16_t IsInvariant : 1;
   };
   enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
 
   class LSBaseSDNodeBitfields {
     friend class LSBaseSDNode;
+    friend class VariableLengthLoadStoreSDNode;
     friend class MaskedLoadStoreSDNode;
     friend class MaskedGatherScatterSDNode;
 
     uint16_t : NumMemSDNodeBits;
 
     // This storage is shared between disparate class hierarchies to hold an
     // enumeration specific to the class hierarchy in use.
     //   LSBaseSDNode => enum ISD::MemIndexedMode
+    //   VariableLengthLoadStoreBaseSDNode => enum ISD::MemIndexedMode
     //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
     //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
     uint16_t AddressingMode : 3;
   };
   enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
 
   class LoadSDNodeBitfields {
     friend class LoadSDNode;
+    friend class VariableLengthLoadSDNode;
     friend class MaskedLoadSDNode;
 
     uint16_t : NumLSBaseSDNodeBits;
 
     uint16_t ExtTy : 2; // enum ISD::LoadExtType
     uint16_t IsExpanding : 1;
   };
 
   class StoreSDNodeBitfields {
     friend class StoreSDNode;
+    friend class VariableLengthStoreSDNode;
     friend class MaskedStoreSDNode;
 
     uint16_t : NumLSBaseSDNodeBits;
 
     uint16_t IsTruncating : 1;
     uint16_t IsCompressing : 1;
   };
 
public:
   const SDValue &getBasePtr() const { return getOperand(2); }
   const SDValue &getOffset() const { return getOperand(3); }
 
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::STORE;
   }
 };
 
+/// This base class is used to represent VARLEN_LOAD and VARLEN_STORE nodes
+class VariableLengthLoadStoreSDNode : public MemSDNode {
+public:
+  friend class SelectionDAG;
+
+  VariableLengthLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
+                                const DebugLoc &dl, SDVTList VTs,
+                                ISD::MemIndexedMode AM, EVT MemVT,
+                                MachineMemOperand *MMO)
+      : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+    LSBaseSDNodeBits.AddressingMode = AM;
+    assert(getAddressingMode() == AM && "Value truncated");
+  }
+
+  // VariableLengthLoadSDNode (Chain, ptr, offset, length, passthru)
+  // VariableLengthStoreSDNode (Chain, data, ptr, offset, length)
+  // Length is an unsigned int.
+  const SDValue &getBasePtr() const {
+    return getOperand(getOpcode() == ISD::VARLEN_LOAD ? 1 : 2);
+  }
+  const SDValue &getOffset() const {
+    return getOperand(getOpcode() == ISD::VARLEN_LOAD ? 2 : 3);
+  }
+  const SDValue &getLength() const {
+    return getOperand(getOpcode() == ISD::VARLEN_LOAD ? 3 : 4);
+  }
+
+  /// Return the addressing mode for this load or store:
+  /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
+  ISD::MemIndexedMode getAddressingMode() const {
+    return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
+  }
+
+  /// Return true if this is a pre/post inc/dec load/store.
+  bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
+
+  /// Return true if this is NOT a pre/post inc/dec load/store.
+  bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::VARLEN_LOAD ||
+           N->getOpcode() == ISD::VARLEN_STORE;
+  }
+};
+
+/// This class is used to represent an VARLEN_LOAD node
+class VariableLengthLoadSDNode : public VariableLengthLoadStoreSDNode {
+public:
+  friend class SelectionDAG;
+
+  VariableLengthLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
+                           ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
+                           bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
+      : VariableLengthLoadStoreSDNode(ISD::VARLEN_LOAD, Order, dl, VTs, AM,
+                                      MemVT, MMO) {
+    LoadSDNodeBits.ExtTy = ETy;
+    LoadSDNodeBits.IsExpanding = IsExpanding;
+  }
+
+  ISD::LoadExtType getExtensionType() const {
+    return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
+  }
+
+  const SDValue &getBasePtr() const { return getOperand(1); }
+  const SDValue &getOffset() const { return getOperand(2); }
+  const SDValue &getLength() const { return getOperand(3); }
+  const SDValue &getPassThru() const { return getOperand(4); }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::VARLEN_LOAD;
+  }
+
+  bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
+};
+
+/// This class is used to represent a VARLEN_STORE node
+class VariableLengthStoreSDNode : public VariableLengthLoadStoreSDNode {
+public:
+  friend class SelectionDAG;
+
+  VariableLengthStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
+                            ISD::MemIndexedMode AM, bool IsTruncating,
+                            EVT MemVT, MachineMemOperand *MMO)
+      : VariableLengthLoadStoreSDNode(ISD::VARLEN_STORE, Order, dl, VTs, AM,
+                                      MemVT, MMO) {
+    StoreSDNodeBits.IsTruncating = IsTruncating;
+  }
+
+  const SDValue &getValue() const { return getOperand(1); }
+  const SDValue &getBasePtr() const { return getOperand(2); }
+  const SDValue &getOffset() const { return getOperand(3); }
+  const SDValue &getLength() const { return getOperand(4); }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::VARLEN_STORE;
+  }
+
+  bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
+};
+
 /// This base class is used to represent MLOAD and MSTORE nodes
 class MaskedLoadStoreSDNode : public MemSDNode {
 public:
   friend class SelectionDAG;
 
   MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                         const DebugLoc &dl, SDVTList VTs,
                         ISD::MemIndexedMode AM, EVT MemVT,

llvm/include/llvm/IR/IRBuilder.h

public:
   /// If the pointer isn't i8* it will be converted.
   CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
 
   /// Create a call to invariant.start intrinsic.
   ///
   /// If the pointer isn't i8* it will be converted.
   CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
 
+  /// Create a call to Variable-Length Load intrinsic
+  CallInst *CreateVariableLengthLoad(Value *Ptr, Align Alignment, Value *Length,
+                                     Value *PassThru = nullptr,
+                                     const Twine &Name = "");
+
+  /// Create a call to Variable-Length Store intrinsic
+  CallInst *CreateVariableLengthStore(Value *Val, Value *Ptr, Align Alignment,
+                                      Value *Length);
+
   /// Create a call to Masked Load intrinsic
   LLVM_ATTRIBUTE_DEPRECATED(
       CallInst *CreateMaskedLoad(Value *Ptr, unsigned Alignment, Value *Mask,
                                  Value *PassThru = nullptr,
                                  const Twine &Name = ""),
       "Use the version that takes Align instead") {
     return CreateMaskedLoad(Ptr, assumeAligned(Alignment), Mask, PassThru,
                             Name);

llvm/include/llvm/IR/Intrinsics.td

 
 }
 
 def int_get_active_lane_mask:
   Intrinsic<[llvm_anyvector_ty],
             [llvm_anyint_ty, LLVMMatchType<1>],
             [IntrNoMem, IntrNoSync, IntrWillReturn]>;
 
+//===-------------------- Variable-Length Intrinsics --------------------===//
+def int_variable_length_load
+    : Intrinsic<[llvm_anyvector_ty],
+      [ LLVMAnyPointerType<LLVMMatchType<0>>,
+      llvm_i32_ty,
+        llvm_anyint_ty,
+        LLVMMatchType<0>,
+      ],
+      [ IntrArgMemOnly, IntrWillReturn,
+        ImmArg<ArgIndex<1>>,
+        IntrReadMem, ]>;
+
+def int_variable_length_store
+    : Intrinsic<[],
+      [ llvm_anyvector_ty,
+        LLVMAnyPointerType<LLVMMatchType<0>>,
+        llvm_i32_ty,
+        llvm_anyint_ty ],
+      [ IntrArgMemOnly,
+        ImmArg<ArgIndex<2>>,
+        IntrWriteMem, ]>;
+
+
 //===-------------------------- Masked Intrinsics -------------------------===//
 //
 def int_masked_store : Intrinsic<[], [llvm_anyvector_ty,
                                       LLVMAnyPointerType<LLVMMatchType<0>>,
                                       llvm_i32_ty,
                                       LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                  [IntrArgMemOnly, IntrWillReturn, ImmArg<ArgIndex<2>>]>;
 

llvm/include/llvm/InitializePasses.h

 void initializeSCEVAAWrapperPassPass(PassRegistry&);
 void initializeSLPVectorizerPass(PassRegistry&);
 void initializeSROALegacyPassPass(PassRegistry&);
 void initializeSafeStackLegacyPassPass(PassRegistry&);
 void initializeSafepointIRVerifierPass(PassRegistry&);
 void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&);
 void initializeModuleSanitizerCoverageLegacyPassPass(PassRegistry &);
 void initializeScalarEvolutionWrapperPassPass(PassRegistry&);
+void initializeScalarizeVariableLengthMemIntrinPass(PassRegistry &);
 void initializeScalarizeMaskedMemIntrinPass(PassRegistry&);
 void initializeScalarizerLegacyPassPass(PassRegistry&);
 void initializeScavengerTestPass(PassRegistry&);
 void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&);
 void initializeSeparateConstOffsetFromGEPPass(PassRegistry&);
 void initializeShadowStackGCLoweringPass(PassRegistry&);
 void initializeShrinkWrapPass(PassRegistry&);
 void initializeSimpleInlinerPass(PassRegistry&);

llvm/include/llvm/LinkAllPasses.h

ForcePassLinking() {
       (void) llvm::createSpeculativeExecutionIfHasBranchDivergencePass();
       (void) llvm::createRewriteSymbolsPass();
       (void) llvm::createStraightLineStrengthReducePass();
       (void) llvm::createMemDerefPrinter();
       (void) llvm::createMustExecutePrinter();
       (void) llvm::createMustBeExecutedContextPrinter();
       (void) llvm::createFloat2IntPass();
       (void) llvm::createEliminateAvailableExternallyPass();
+      (void)llvm::createScalarizeVariableLengthMemIntrinPass();
       (void) llvm::createScalarizeMaskedMemIntrinPass();
       (void) llvm::createWarnMissedTransformationsPass();
       (void) llvm::createHardwareLoopsPass();
       (void) llvm::createInjectTLIMappingsLegacyPass();
       (void) llvm::createUnifyLoopExitsPass();
       (void) llvm::createFixIrreduciblePass();
 
       (void)new llvm::IntervalPartition();

llvm/include/llvm/Target/TargetSelectionDAG.td

 def SDTStore : SDTypeProfile<0, 2, [        // store
   SDTCisPtrTy<1>
 ]>;
 
 def SDTIStore : SDTypeProfile<1, 3, [       // indexed store
   SDTCisSameAs<0, 2>, SDTCisPtrTy<0>, SDTCisPtrTy<3>
 ]>;
 
+def SDTVariableLengthLoad: SDTypeProfile<1, 4, [       // partial load
+  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisInt<3>, SDTCisSameAs<0, 4>,
+]>;
+def SDTVariableLengthStore: SDTypeProfile<0, 4, [       // partial store
+  SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisInt<3>
+]>;
+
 def SDTMaskedStore: SDTypeProfile<0, 4, [       // masked store
   SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisSameNumEltsAs<0, 3>
 ]>;
 
 def SDTMaskedLoad: SDTypeProfile<1, 4, [       // masked load
   SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVec<3>, SDTCisSameAs<0, 4>,
   SDTCisSameNumEltsAs<0, 3>
 ]>;

 def atomic_load_fsub : SDNode<"ISD::ATOMIC_LOAD_FSUB" , SDTFPAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 
 def atomic_load      : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,
                     [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def atomic_store     : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,
                     [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 
+def varlen_ld    : SDNode<"ISD::VARLEN_LOAD",  SDTVariableLengthLoad,
+                       [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def varlen_st    : SDNode<"ISD::VARLEN_STORE",  SDTVariableLengthStore,
+                       [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
 def masked_st    : SDNode<"ISD::MSTORE",  SDTMaskedStore,
                        [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def masked_ld    : SDNode<"ISD::MLOAD",  SDTMaskedLoad,
                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
 // Do not use ld, st directly. Use load, extload, sextload, zextload, store,
 // and truncst (see below).
 def ld         : SDNode<"ISD::LOAD"       , SDTLoad,

llvm/lib/Analysis/TargetTransformInfo.cpp

 bool TargetTransformInfo::shouldFavorPostInc() const {
   return TTIImpl->shouldFavorPostInc();
 }
 
 bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const {
   return TTIImpl->shouldFavorBackedgeIndex(L);
 }
 
+bool TargetTransformInfo::isLegalVariableLengthLoad(Type *DataType,
+                                                    Align Alignment,
+                                                    Type *LengthType) const {
+  return TTIImpl->isLegalVariableLengthLoad(DataType, Alignment, LengthType);
+}
+
+bool TargetTransformInfo::isLegalVariableLengthStore(Type *DataType,
+                                                     Align Alignment,
+                                                     Type *LengthType) const {
+  return TTIImpl->isLegalVariableLengthStore(DataType, Alignment, LengthType);
+}
+
 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
                                              Align Alignment) const {
   return TTIImpl->isLegalMaskedStore(DataType, Alignment);
 }
 
 bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
                                             Align Alignment) const {
   return TTIImpl->isLegalMaskedLoad(DataType, Alignment);

llvm/lib/CodeGen/CMakeLists.txt

add_llvm_component_library(LLVMCodeGen
   MIRNamerPass.cpp
   MIRCanonicalizerPass.cpp
   RegisterUsageInfo.cpp
   RegUsageInfoCollector.cpp
   RegUsageInfoPropagate.cpp
   ResetMachineFunctionPass.cpp
   SafeStack.cpp
   SafeStackLayout.cpp
+  ScalarizeVariableLengthMemIntrin.cpp
   ScalarizeMaskedMemIntrin.cpp
   ScheduleDAG.cpp
   ScheduleDAGInstrs.cpp
   ScheduleDAGPrinter.cpp
   ScoreboardHazardRecognizer.cpp
   ShadowStackGCLowering.cpp
   ShrinkWrap.cpp
   SjLjEHPrepare.cpp

llvm/lib/CodeGen/CodeGen.cpp

void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeRABasicPass(Registry);
   initializeRAGreedyPass(Registry);
   initializeRegAllocFastPass(Registry);
   initializeRegUsageInfoCollectorPass(Registry);
   initializeRegUsageInfoPropagationPass(Registry);
   initializeRegisterCoalescerPass(Registry);
   initializeRenameIndependentSubregsPass(Registry);
   initializeSafeStackLegacyPassPass(Registry);
+  initializeScalarizeVariableLengthMemIntrinPass(Registry);
   initializeScalarizeMaskedMemIntrinPass(Registry);
   initializeShrinkWrapPass(Registry);
   initializeSjLjEHPreparePass(Registry);
   initializeSlotIndexesPass(Registry);
   initializeStackColoringPass(Registry);
   initializeStackMapLivenessPass(Registry);
   initializeStackProtectorPass(Registry);
   initializeStackSlotColoringPass(Registry);

llvm/lib/CodeGen/ScalarizeVariableLengthMemIntrin.cpp

+//===- ScalarizeVariableLengthMemIntrin.cpp - Scalarize unsupported
+//variable-length mem --===//
+//                                    instrinsics
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass replaces variable-length memory intrinsics - when unsupported by
+// the target
+// - with a chain of basic blocks, that deal with power-of-two subvectors
+// one-by-one if the appropriate bits are set in the length.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Casting.h"
+#include <algorithm>
+#include <cassert>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "scalarize-varlen-mem-intrin"
+
+namespace {
+
+class ScalarizeVariableLengthMemIntrin : public FunctionPass {
+  const TargetTransformInfo *TTI = nullptr;
+  const DataLayout *DL = nullptr;
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+
+  explicit ScalarizeVariableLengthMemIntrin() : FunctionPass(ID) {
+    initializeScalarizeVariableLengthMemIntrinPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override;
+
+  StringRef getPassName() const override {
+    return "Scalarize VariableLength Memory Intrinsics";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<TargetTransformInfoWrapperPass>();
+  }
+
+private:
+  bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT);
+  bool optimizeCallInst(CallInst *CI, bool &ModifiedDT);
+};
+
+} // end anonymous namespace
+
+char ScalarizeVariableLengthMemIntrin::ID = 0;
+
+INITIALIZE_PASS(ScalarizeVariableLengthMemIntrin, DEBUG_TYPE,
+                "Scalarize unsupported variable-length memory intrinsics",
+                false, false)
+
+FunctionPass *llvm::createScalarizeVariableLengthMemIntrinPass() {
+  return new ScalarizeVariableLengthMemIntrin();
+}
+
+static void scalarizeVariableLengthLoad(CallInst *CI, bool &ModifiedDT) {
+  Value *Ptr = CI->getArgOperand(0);
+  const Align Alignment =
+      cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue();
+  Value *Length = CI->getArgOperand(2);
+  Value *Pass = CI->getArgOperand(3);
+
+  VectorType *VecType = cast<VectorType>(CI->getType());
+  unsigned VectorWidth = VecType->getNumElements();
+  Type *EltTy = VecType->getElementType();
+  Type *LengthType = Length->getType();
+  unsigned PtrAS = Ptr->getType()->getPointerAddressSpace();
+
+  IRBuilder<> Builder(CI);
+  Value *BasePtr = Builder.CreateBitCast(Ptr, EltTy->getPointerTo(PtrAS));
+  Value *VResult = Pass;
+  Builder.SetCurrentDebugLocation(CI->getDebugLoc());
+
+  if (isa<Constant>(Length)) {
+    Value *VResult = Builder.CreateAlignedLoad(VecType, Ptr, Alignment);
+    CI->replaceAllUsesWith(VResult);
+    CI->eraseFromParent();
+    return;
+  }
+
+  unsigned LastBranchBit =
+      findLastSet(VectorWidth); // note that bits are 0-indexed
+  unsigned BranchMask = maskTrailingOnes<unsigned>(LastBranchBit + 1);
+  for (unsigned BranchBit = LastBranchBit; BranchBit--;) {
+    unsigned BranchOffsetMask =
+        maskTrailingOnes<unsigned>(BranchBit + 1) ^ BranchMask;
+    unsigned BranchWidth = 1 << BranchBit;
+    Value *BranchWidthValue = ConstantInt::get(LengthType, BranchWidth);
+    Value *BranchOffsetMaskValue =
+        ConstantInt::get(LengthType, BranchOffsetMask);
+
+    BasicBlock *BB = CI->getParent();
+    BasicBlock *ContBB = BB->splitBasicBlock(CI, "cont");
+    BasicBlock *BranchBB = ContBB->splitBasicBlock(CI, "cond.load");
+
+    Instruction *OldTerm = BB->getTerminator();
+    Value *BranchPredicate = Builder.CreateICmpUGE(Length, BranchWidthValue);
+    BranchInst::Create(BranchBB, ContBB, BranchPredicate, OldTerm);
+    OldTerm->eraseFromParent();
+
+    //    auto *BranchTy = VectorType::get(EltTy, BranchWidth);
+    //    auto *BranchPtrTy = BranchTy->getPointerTo(PtrAS);
+    const Align BranchAlignment =
+        commonAlignment(Alignment, EltTy->getPrimitiveSizeInBits() / 8);
+
+    Builder.SetInsertPoint(BranchBB->getTerminator());
+    Value *BranchOffsetValue = Builder.CreateAnd(Length, BranchOffsetMaskValue);
+    //    Value *BranchPtr = Builder.CreatePointerCast(
+    //        Builder.CreateInBoundsGEP(BasePtr, BranchOffsetValue),
+    //        BranchPtrTy);
+    Value *BranchPtr = Builder.CreateInBoundsGEP(BasePtr, BranchOffsetValue);
+
+    // todo: investigate possibilities for vectorizing variable-offset
+    // insertelements
+    Value *BranchRes = VResult;
+    for (unsigned i = 0; i < BranchWidth; ++i) {
+      auto *vi = ConstantInt::get(LengthType, i);
+      Value *BranchLoad = Builder.CreateAlignedLoad(
+          BasePtr->getType(), Builder.CreateInBoundsGEP(BranchPtr, vi),
+          BranchAlignment);
+      BranchRes = Builder.CreateInsertElement(
+          BranchRes, BranchLoad, Builder.CreateAdd(BranchOffsetValue, vi));
+    }
+
+    Builder.SetInsertPoint(CI);
+    PHINode *Phi = Builder.CreatePHI(VecType, 2);
+    Phi->addIncoming(BranchRes, BranchBB);
+    Phi->addIncoming(VResult, BB);
+    VResult = Phi;
+  }
+  CI->replaceAllUsesWith(VResult);
+  CI->eraseFromParent();
+  ModifiedDT = true;
+  return;
+}
+
+static void scalarizeVariableLengthStore(CallInst *CI, bool &ModifiedDT) {
+  Value *Src = CI->getArgOperand(0);
+  Value *Ptr = CI->getArgOperand(1);
+  const Align Alignment =
+      cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue();
+  Value *Length = CI->getArgOperand(3);
+
+  VectorType *VecType = cast<VectorType>(Src->getType());
+  unsigned VectorWidth = VecType->getNumElements();
+  Type *EltTy = VecType->getElementType();
+  Type *LengthType = Length->getType();
+  unsigned PtrAS = Ptr->getType()->getPointerAddressSpace();
+
+  IRBuilder<> Builder(CI);
+  Value *BasePtr = Builder.CreateBitCast(Ptr, EltTy->getPointerTo(PtrAS));
+  Builder.SetCurrentDebugLocation(CI->getDebugLoc());
+
+  if (isa<Constant>(Length)) {
+    Builder.CreateAlignedStore(Src, Ptr, Alignment);
+    CI->eraseFromParent();
+    return;
+  }
+
+  unsigned LastBranchBit =
+      findLastSet(VectorWidth); // note that bits are 0-indexed
+  unsigned BranchMask = maskTrailingOnes<unsigned>(LastBranchBit + 1);
+  for (unsigned BranchBit = LastBranchBit; BranchBit--;) {
+    unsigned BranchOffsetMask =
+        maskTrailingOnes<unsigned>(BranchBit + 1) ^ BranchMask;
+    unsigned BranchWidth = 1 << BranchBit;
+    Value *BranchWidthValue = ConstantInt::get(LengthType, BranchWidth);
+    Value *BranchOffsetMaskValue =
+        ConstantInt::get(LengthType, BranchOffsetMask);
+
+    BasicBlock *BB = CI->getParent();
+    BasicBlock *ContBB = BB->splitBasicBlock(CI, "cont");
+    BasicBlock *BranchBB = ContBB->splitBasicBlock(CI, "cond.store");
+
+    Instruction *OldTerm = BB->getTerminator();
+    Value *BranchPredicate = Builder.CreateICmpUGE(Length, BranchWidthValue);
+    BranchInst::Create(BranchBB, ContBB, BranchPredicate, OldTerm);
+    OldTerm->eraseFromParent();
+
+    //    auto *BranchTy = VectorType::get(EltTy, BranchWidth);
+    //    auto *BranchPtrTy = BranchTy->getPointerTo(PtrAS);
+    const Align BranchAlignment =
+        commonAlignment(Alignment, EltTy->getPrimitiveSizeInBits() / 8);
+
+    Builder.SetInsertPoint(BranchBB->getTerminator());
+    Value *BranchOffsetValue = Builder.CreateAnd(Length, BranchOffsetMaskValue);
+    //    Value *BranchPtr = Builder.CreatePointerCast(
+    //        Builder.CreateInBoundsGEP(BasePtr, BranchOffsetValue),
+    //        BranchPtrTy);
+    Value *BranchPtr = Builder.CreateInBoundsGEP(BasePtr, BranchOffsetValue);
+
+    // todo: investigate possibilities for vectorizing variable-offset
+    // extractelements
+    for (unsigned i = 0; i < BranchWidth; ++i) {
+      auto *vi = ConstantInt::get(LengthType, i);
+      Value *BranchLoad = Builder.CreateExtractElement(
+          Src, Builder.CreateAdd(BranchOffsetValue, vi));
+      Builder.CreateAlignedStore(
+          BranchLoad,
+          Builder.CreateInBoundsGEP(BasePtr->getType(), BranchPtr, vi),
+          BranchAlignment);
+    }
+  }
+  CI->eraseFromParent();
+  ModifiedDT = true;
+  return;
+}
+
+bool ScalarizeVariableLengthMemIntrin::runOnFunction(Function &F) {
+  bool EverMadeChange = false;
+
+  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+  DL = &F.getParent()->getDataLayout();
+
+  bool MadeChange = true;
+  while (MadeChange) {
+    MadeChange = false;
+    for (Function::iterator I = F.begin(); I != F.end();) {
+      BasicBlock *BB = &*I++;
+      bool ModifiedDTOnIteration = false;
+      MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration);
+
+      // Restart BB iteration if the dominator tree of the Function was changed
+      if (ModifiedDTOnIteration)
+        break;
+    }
+
+    EverMadeChange |= MadeChange;
+  }
+
+  return EverMadeChange;
+}
+
+bool ScalarizeVariableLengthMemIntrin::optimizeBlock(BasicBlock &BB,
+                                                     bool &ModifiedDT) {
+  bool MadeChange = false;
+
+  BasicBlock::iterator CurInstIterator = BB.begin();
+  while (CurInstIterator != BB.end()) {
+    if (CallInst *CI = dyn_cast<CallInst>(&*CurInstIterator++))
+      MadeChange |= optimizeCallInst(CI, ModifiedDT);
+    if (ModifiedDT)
+      return true;
+  }
+
+  return MadeChange;
+}
+
+bool ScalarizeVariableLengthMemIntrin::optimizeCallInst(CallInst *CI,
+                                                        bool &ModifiedDT) {
+  IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
+  if (II) {
+    switch (II->getIntrinsicID()) {
+    default:
+      break;
+    case Intrinsic::variable_length_load:
+      // Scalarize unsupported vector variable-length load
+      if (TTI->isLegalVariableLengthLoad(
+              CI->getType(),
+              cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue(),
+              CI->getArgOperand(2)->getType()))
+        return false;
+      scalarizeVariableLengthLoad(CI, ModifiedDT);
+      return true;
+    case Intrinsic::variable_length_store:
+      if (TTI->isLegalVariableLengthStore(
+              CI->getArgOperand(0)->getType(),
+              cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue(),
+              CI->getArgOperand(3)->getType()))
+        return false;
+      scalarizeVariableLengthStore(CI, ModifiedDT);
+      return true;
+    }
+  }
+
+  return false;
+}

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

private:
     SDValue visitINSERT_VECTOR_ELT(SDNode *N);
     SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
     SDValue visitBUILD_VECTOR(SDNode *N);
     SDValue visitCONCAT_VECTORS(SDNode *N);
     SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
     SDValue visitVECTOR_SHUFFLE(SDNode *N);
     SDValue visitSCALAR_TO_VECTOR(SDNode *N);
     SDValue visitINSERT_SUBVECTOR(SDNode *N);
+    SDValue visitVARLEN_STORE(SDNode *N);
+    SDValue visitVARLEN_LOAD(SDNode *N);
     SDValue visitMLOAD(SDNode *N);
     SDValue visitMSTORE(SDNode *N);
     SDValue visitMGATHER(SDNode *N);
     SDValue visitMSCATTER(SDNode *N);
     SDValue visitFP_TO_FP16(SDNode *N);
     SDValue visitFP16_TO_FP(SDNode *N);
     SDValue visitVECREDUCE(SDNode *N);
 
SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
   case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
   case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
   case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
   case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
   case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
   case ISD::SCALAR_TO_VECTOR:   return visitSCALAR_TO_VECTOR(N);
   case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
+  case ISD::VARLEN_LOAD:
+    return visitVARLEN_LOAD(N);
+  case ISD::VARLEN_STORE:
+    return visitVARLEN_STORE(N);
   case ISD::MGATHER:            return visitMGATHER(N);
   case ISD::MLOAD:              return visitMLOAD(N);
   case ISD::MSCATTER:           return visitMSCATTER(N);
   case ISD::MSTORE:             return visitMSTORE(N);
   case ISD::LIFETIME_END:       return visitLIFETIME_END(N);
   case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
   case ISD::FP16_TO_FP:         return visitFP16_TO_FP(N);
   case ISD::FREEZE:             return visitFREEZE(N);
SDValue DAGCombiner::visitMGATHER(SDNode *N) {
 
   // Zap gathers with a zero mask.
   if (ISD::isBuildVectorAllZeros(Mask.getNode()))
     return CombineTo(N, MGT->getPassThru(), MGT->getChain());
 
   return SDValue();
 }
 
+SDValue DAGCombiner::visitVARLEN_LOAD(SDNode *N) {
+  // Try transforming N to an indexed load.
+  if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
+    return SDValue(N, 0);
+  return SDValue();
+}
+
+SDValue DAGCombiner::visitVARLEN_STORE(SDNode *N) {
+  // Try transforming N to an indexed store.
+  if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
+    return SDValue(N, 0);
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitMLOAD(SDNode *N) {
   MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
   SDValue Mask = MLD->getMask();
   SDLoc DL(N);
 
   // Zap masked loads with a zero mask.
   if (ISD::isBuildVectorAllZeros(Mask.getNode()))
     return CombineTo(N, MLD->getPassThru(), MLD->getChain());

llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

#endif
   case ISD::SDIVFIXSAT:
   case ISD::UDIVFIX:
   case ISD::UDIVFIXSAT: {
     unsigned Scale = Node->getConstantOperandVal(2);
     Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
                                               Node->getValueType(0), Scale);
     break;
   }
+  case ISD::VARLEN_STORE:
+    Action = TLI.getOperationAction(
+        Node->getOpcode(),
+        cast<VariableLengthStoreSDNode>(Node)->getValue().getValueType());
+    break;
   case ISD::MSCATTER:
     Action = TLI.getOperationAction(Node->getOpcode(),
                     cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
     break;
   case ISD::MSTORE:
     Action = TLI.getOperationAction(Node->getOpcode(),
                     cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
     break;

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

#endif
   case ISD::CTLZ_ZERO_UNDEF:
   case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
   case ISD::CTPOP:       Res = PromoteIntRes_CTPOP(N); break;
   case ISD::CTTZ_ZERO_UNDEF:
   case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
   case ISD::EXTRACT_VECTOR_ELT:
                          Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
   case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
+  case ISD::VARLEN_LOAD:
+    Res = PromoteIntRes_VARLEN_LOAD(cast<VariableLengthLoadSDNode>(N));
+    break;
   case ISD::MLOAD:       Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
     break;
   case ISD::MGATHER:     Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
     break;
   case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
   case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
   case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
   case ISD::STRICT_FSETCC:
SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
                                N->getMemoryVT(), N->getMemOperand());
 
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
   return Res;
 }
 
+SDValue
+DAGTypeLegalizer::PromoteIntRes_VARLEN_LOAD(VariableLengthLoadSDNode *N) {
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
+
+  SDLoc dl(N);
+  SDValue Res = DAG.getVariableLengthLoad(
+      NVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), N->getLength(),
+      ExtPassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
+      ISD::EXTLOAD);
+  // Legalize the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+  return Res;
+}
+
 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
 
   SDLoc dl(N);
   SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
                                   N->getOffset(), N->getMask(), ExtPassThru,
                                   N->getMemoryVT(), N->getMemOperand(),
bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
   case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
   case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
   case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
   case ISD::SIGN_EXTEND:  Res = PromoteIntOp_SIGN_EXTEND(N); break;
   case ISD::SINT_TO_FP:   Res = PromoteIntOp_SINT_TO_FP(N); break;
   case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
   case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
                                                    OpNo); break;
+  case ISD::VARLEN_LOAD:
+    Res = PromoteIntOp_VARLEN_LOAD(cast<VariableLengthLoadSDNode>(N), OpNo);
+    break;
+  case ISD::VARLEN_STORE:
+    Res = PromoteIntOp_VARLEN_STORE(cast<VariableLengthStoreSDNode>(N), OpNo);
+    break;
   case ISD::MSTORE:       Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
                                                     OpNo); break;
   case ISD::MLOAD:        Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
                                                     OpNo); break;
   case ISD::MGATHER:  Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
                                                  OpNo); break;
   case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
                                                   OpNo); break;
SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
 
   SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.
 
   // Truncate the value and store the result.
   return DAG.getTruncStore(Ch, dl, Val, Ptr,
                            N->getMemoryVT(), N->getMemOperand());
 }
 
+SDValue DAGTypeLegalizer::PromoteIntOp_VARLEN_LOAD(VariableLengthLoadSDNode *N,
+                                                   unsigned OpNo) {
+  assert(OpNo == 3 && "Only know how to promote the length!");
+  SDValue Length = GetPromotedInteger(N->getOperand(3));
+  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
+  NewOps[OpNo] = Length;
+  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+}
+
+SDValue
+DAGTypeLegalizer::PromoteIntOp_VARLEN_STORE(VariableLengthStoreSDNode *N,
+                                            unsigned OpNo) {
+  SDValue DataOp = N->getValue();
+  SDValue Length = N->getLength();
+  SDLoc dl(N);
+
+  bool TruncateStore = false;
+  if (OpNo == 4) {
+    Length = GetPromotedInteger(Length);
+    // Update in place.
+    SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
+    NewOps[4] = Length;
+    return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
+  } else { // Data operand
+    assert(OpNo == 1 && "Unexpected operand for promotion");
+    DataOp = GetPromotedInteger(DataOp);
+    TruncateStore = true;
+  }
+
+  return DAG.getVariableLengthStore(N->getChain(), dl, DataOp, N->getBasePtr(),
+                                    N->getOffset(), Length, N->getMemoryVT(),
+                                    N->getMemOperand(), N->getAddressingMode(),
+                                    TruncateStore);
+}
+
 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
                                               unsigned OpNo) {
 
   SDValue DataOp = N->getValue();
   EVT DataVT = DataOp.getValueType();
   SDValue Mask = N->getMask();
   SDLoc dl(N);
 

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   SDValue PromoteIntRes_CTPOP(SDNode *N);
   SDValue PromoteIntRes_CTTZ(SDNode *N);
   SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
   SDValue PromoteIntRes_FP_TO_FP16(SDNode *N);
   SDValue PromoteIntRes_FREEZE(SDNode *N);
   SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
   SDValue PromoteIntRes_LOAD(LoadSDNode *N);
+  SDValue PromoteIntRes_VARLEN_LOAD(VariableLengthLoadSDNode *N);
   SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
   SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
   SDValue PromoteIntRes_Overflow(SDNode *N);
   SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_SELECT(SDNode *N);
   SDValue PromoteIntRes_VSELECT(SDNode *N);
   SDValue PromoteIntRes_SELECT_CC(SDNode *N);
   SDValue PromoteIntRes_SETCC(SDNode *N);
private:
   SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
   SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
   SDValue PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N);
   SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_TRUNCATE(SDNode *N);
   SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
   SDValue PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N);
   SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
+  SDValue PromoteIntOp_VARLEN_LOAD(VariableLengthLoadSDNode *N, unsigned OpNo);
+  SDValue PromoteIntOp_VARLEN_STORE(VariableLengthStoreSDNode *N,
+                                    unsigned OpNo);
   SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
   SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FIX(SDNode *N);
private:
   SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
   SDValue WidenVecRes_BITCAST(SDNode* N);
   SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
   SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
   SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
   SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
   SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
   SDValue WidenVecRes_LOAD(SDNode* N);
+  SDValue WidenVecRes_VARLEN_LOAD(VariableLengthLoadSDNode *N);
   SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
   SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
   SDValue WidenVecRes_ScalarOp(SDNode* N);
   SDValue WidenVecRes_SELECT(SDNode* N);
   SDValue WidenVSELECTMask(SDNode *N);
   SDValue WidenVecRes_SELECT_CC(SDNode* N);
   SDValue WidenVecRes_SETCC(SDNode* N);
   SDValue WidenVecRes_STRICT_FSETCC(SDNode* N);
private:
   // Widen Vector Operand.
   bool WidenVectorOperand(SDNode *N, unsigned OpNo);
   SDValue WidenVecOp_BITCAST(SDNode *N);
   SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
   SDValue WidenVecOp_EXTEND(SDNode *N);
   SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue WidenVecOp_STORE(SDNode* N);
+  SDValue WidenVecOp_VARLEN_STORE(SDNode *N, unsigned OpNo);
   SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_SETCC(SDNode* N);
   SDValue WidenVecOp_STRICT_FSETCC(SDNode* N);
   SDValue WidenVecOp_VSELECT(SDNode *N);
 
   SDValue WidenVecOp_Convert(SDNode *N);

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

#endif
   case ISD::VSELECT:
   case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
   case ISD::SELECT_CC:         Res = WidenVecRes_SELECT_CC(N); break;
   case ISD::SETCC:             Res = WidenVecRes_SETCC(N); break;
   case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
   case ISD::VECTOR_SHUFFLE:
     Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
     break;
+  case ISD::VARLEN_LOAD:
+    Res = WidenVecRes_VARLEN_LOAD(cast<VariableLengthLoadSDNode>(N));
+    break;
   case ISD::MLOAD:
     Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
     break;
   case ISD::MGATHER:
     Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
     break;
 
   case ISD::ADD:
SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
 
   // Modified the chain - switch anything that used the old chain to use
   // the new one.
   ReplaceValueWith(SDValue(N, 1), NewChain);
 
   return Result;
 }
 
+SDValue DAGTypeLegalizer::WidenVecRes_VARLEN_LOAD(VariableLengthLoadSDNode *N) {
+  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  SDValue PassThru = GetWidenedVector(N->getPassThru());
+  ISD::LoadExtType ExtType = N->getExtensionType();
+  SDLoc dl(N);
+
+  SDValue Res = DAG.getVariableLengthLoad(
+      WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(),
+      N->getLength(), PassThru, N->getMemoryVT(), N->getMemOperand(),
+      N->getAddressingMode(), ExtType, N->isExpandingLoad());
+  // Legalize the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+  return Res;
+}
+
 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
 
   EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
   SDValue Mask = N->getMask();
   EVT MaskVT = Mask.getValueType();
   SDValue PassThru = GetWidenedVector(N->getPassThru());
   ISD::LoadExtType ExtType = N->getExtensionType();
   SDLoc dl(N);

 #endif
     llvm_unreachable("Do not know how to widen this operator's operand!");
 
   case ISD::BITCAST:            Res = WidenVecOp_BITCAST(N); break;
   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
   case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
+  case ISD::VARLEN_STORE:
+    Res = WidenVecOp_VARLEN_STORE(N, OpNo);
+    break;
   case ISD::MSTORE:             Res = WidenVecOp_MSTORE(N, OpNo); break;
   case ISD::MGATHER:            Res = WidenVecOp_MGATHER(N, OpNo); break;
   case ISD::MSCATTER:           Res = WidenVecOp_MSCATTER(N, OpNo); break;
   case ISD::SETCC:              Res = WidenVecOp_SETCC(N); break;
   case ISD::STRICT_FSETCC:
   case ISD::STRICT_FSETCCS:     Res = WidenVecOp_STRICT_FSETCC(N); break;
   case ISD::VSELECT:            Res = WidenVecOp_VSELECT(N); break;
   case ISD::FCOPYSIGN:          Res = WidenVecOp_FCOPYSIGN(N); break;
else
     GenWidenVectorStores(StChain, ST);
 
   if (StChain.size() == 1)
     return StChain[0];
   else
     return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
 }
 
+SDValue DAGTypeLegalizer::WidenVecOp_VARLEN_STORE(SDNode *N, unsigned OpNo) {
+  assert((OpNo == 1) && "Can widen only data operand of varlen_store");
+  auto *ST = cast<VariableLengthStoreSDNode>(N);
+  SDValue StVal = ST->getValue();
+  SDLoc dl(N);
+
+  StVal = GetWidenedVector(StVal);
+  return DAG.getVariableLengthStore(ST->getChain(), dl, StVal, ST->getBasePtr(),
+                                    ST->getOffset(), ST->getLength(),
+                                    ST->getMemoryVT(), ST->getMemOperand(),
+                                    ST->getAddressingMode(), false);
+}
+
 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
   assert((OpNo == 1 || OpNo == 3) &&
          "Can widen only data or mask operand of mstore");
   MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
   SDValue Mask = MST->getMask();
   EVT MaskVT = Mask.getValueType();
   SDValue StVal = MST->getValue();
   SDLoc dl(N);

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
 
   CSEMap.InsertNode(N, IP);
   InsertNode(N);
   SDValue V(N, 0);
   NewSDValueDbgMsg(V, "Creating new node: ", this);
   return V;
 }
 
+SDValue SelectionDAG::getVariableLengthLoad(
+    EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset,
+    SDValue Length, SDValue PassThru, EVT MemVT, MachineMemOperand *MMO,
+    ISD::MemIndexedMode AM, ISD::LoadExtType ExtTy, bool IsExpanding) {
+  bool Indexed = (AM != ISD::UNINDEXED);
+  assert((Indexed || Offset.isUndef()) &&
+         "Unindexed variable-length load with an offset!");
+  SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
+                         : getVTList(VT, MVT::Other);
+  SDValue Ops[] = {Chain, Base, Offset, Length, PassThru};
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::VARLEN_LOAD, VTs, Ops);
+  ID.AddInteger(MemVT.getRawBits());
+  ID.AddInteger(getSyntheticNodeSubclassData<VariableLengthLoadSDNode>(
+      dl.getIROrder(), VTs, AM, ExtTy, IsExpanding, MemVT, MMO));
+  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+  void *IP = nullptr;
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
+    cast<VariableLengthLoadSDNode>(E)->refineAlignment(MMO);
+    return SDValue(E, 0);
+  }
+  auto *N = newSDNode<VariableLengthLoadSDNode>(dl.getIROrder(),
+                                                dl.getDebugLoc(), VTs, AM,
+                                                ExtTy, IsExpanding, MemVT, MMO);
+  createOperands(N, Ops);
+  CSEMap.InsertNode(N, IP);
+  InsertNode(N);
+  SDValue V(N, 0);
+  NewSDValueDbgMsg(V, "Creating new node: ", this);
+  return V;
+}
+
+SDValue SelectionDAG::getIndexedVariableLengthLoad(SDValue OrigLoad,
+                                                   const SDLoc &dl,
+                                                   SDValue Base, SDValue Offset,
+                                                   ISD::MemIndexedMode AM) {
+  auto *LD = cast<VariableLengthLoadSDNode>(OrigLoad);
+  assert(LD->getOffset().isUndef() &&
+         "Variable-length load is already a indexed load!");
+  return getVariableLengthLoad(OrigLoad.getValueType(), dl, LD->getChain(),
+                               Base, Offset, LD->getLength(), LD->getPassThru(),
+                               LD->getMemoryVT(), LD->getMemOperand(), AM,
+                               LD->getExtensionType(), LD->isExpandingLoad());
+}
+
+SDValue SelectionDAG::getVariableLengthStore(SDValue Chain, const SDLoc &dl,
+                                             SDValue Val, SDValue Base,
+                                             SDValue Offset, SDValue Length,
+                                             EVT MemVT, MachineMemOperand *MMO,
+                                             ISD::MemIndexedMode AM,
+                                             bool IsTruncating) {
+  assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
+  bool Indexed = (AM != ISD::UNINDEXED);
+  assert((Indexed || Offset.isUndef()) &&
+         "Unindexed variable-length store with an offset!");
+  SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
+                         : getVTList(MVT::Other);
+  SDValue Ops[] = {Chain, Val, Base, Offset, Length};
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::VARLEN_STORE, VTs, Ops);
+  ID.AddInteger(MemVT.getRawBits());
+  ID.AddInteger(getSyntheticNodeSubclassData<VariableLengthStoreSDNode>(
+      dl.getIROrder(), VTs, AM, IsTruncating, MemVT, MMO));
+  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
+  void *IP = nullptr;
+  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
+    cast<VariableLengthStoreSDNode>(E)->refineAlignment(MMO);
+    return SDValue(E, 0);
+  }
+  auto *N = newSDNode<VariableLengthStoreSDNode>(
+      dl.getIROrder(), dl.getDebugLoc(), VTs, AM, IsTruncating, MemVT, MMO);
+  createOperands(N, Ops);
+  CSEMap.InsertNode(N, IP);
+  InsertNode(N);
+  SDValue V(N, 0);
+  NewSDValueDbgMsg(V, "Creating new node: ", this);
+  return V;
+}
+
+SDValue SelectionDAG::getIndexedVariableLengthStore(SDValue OrigStore,
+                                                    const SDLoc &dl,
+                                                    SDValue Base,
+                                                    SDValue Offset,
+                                                    ISD::MemIndexedMode AM) {
+  auto *ST = cast<VariableLengthStoreSDNode>(OrigStore);
+  assert(ST->getOffset().isUndef() &&
+         "Variable-length store is already a indexed store!");
+  return getVariableLengthStore(
+      ST->getChain(), dl, ST->getValue(), Base, Offset, ST->getLength(),
+      ST->getMemoryVT(), ST->getMemOperand(), AM, ST->isTruncatingStore());
+}
+
 SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
                                     SDValue Base, SDValue Offset, SDValue Mask,
                                     SDValue PassThru, EVT MemVT,
                                     MachineMemOperand *MMO,
                                     ISD::MemIndexedMode AM,
                                     ISD::LoadExtType ExtTy, bool isExpanding) {
   bool Indexed = AM != ISD::UNINDEXED;
   assert((Indexed || Offset.isUndef()) &&

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h

private:
   void visitLandingPad(const LandingPadInst &LP);
 
   void visitGetElementPtr(const User &I);
   void visitSelect(const User &I);
 
   void visitAlloca(const AllocaInst &I);
   void visitLoad(const LoadInst &I);
   void visitStore(const StoreInst &I);
+  void visitVariableLengthLoad(const CallInst &I, bool IsExpanding = false);
+  void visitVariableLengthStore(const CallInst &I);
   void visitMaskedLoad(const CallInst &I, bool IsExpanding = false);
   void visitMaskedStore(const CallInst &I, bool IsCompressing = false);
   void visitMaskedGather(const CallInst &I);
   void visitMaskedScatter(const CallInst &I);
   void visitAtomicCmpXchg(const AtomicCmpXchgInst &I);
   void visitAtomicRMW(const AtomicRMWInst &I);
   void visitFence(const FenceInst &I);
   void visitPHI(const PHINode &I);

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
     Chains[ChainI] = St;
   }
 
   SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                   makeArrayRef(Chains.data(), ChainI));
   DAG.setRoot(StoreNode);
 }
 
+void SelectionDAGBuilder::visitVariableLengthLoad(const CallInst &I,
+                                                  bool IsExpanding) {
+  SDLoc sdl = getCurSDLoc();
+  unsigned i_op = 0;
+  Value *PtrOperand = I.getArgOperand(i_op++);
+  SDValue Ptr = getValue(PtrOperand);
+  MaybeAlign Alignment;
+  if (IsExpanding)
+    Alignment = None;
+  else
+    Alignment =
+        MaybeAlign(cast<ConstantInt>(I.getArgOperand(i_op++))->getZExtValue());
+  SDValue Length = getValue(I.getArgOperand(i_op++));
+  SDValue Src0 = getValue(I.getArgOperand(i_op++));
+  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
+  EVT VT = Src0.getValueType();
+  if (!Alignment)
+    Alignment = DAG.getEVTAlign(VT);
+  AAMDNodes AAInfo;
+  I.getAAMetadata(AAInfo);
+  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
+
+  // Do not serialize variable-length loads of constant memory with anything.
+  MemoryLocation ML;
+  if (VT.isScalableVector())
+    ML = MemoryLocation(PtrOperand);
+  else
+    ML = MemoryLocation(PtrOperand,
+                        LocationSize::precise(
+                            DAG.getDataLayout().getTypeStoreSize(I.getType())),
+                        AAInfo);
+  bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
+  SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
+      VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo, Ranges);
+  SDValue Load = DAG.getVariableLengthLoad(
+      VT, sdl, InChain, Ptr, Offset, Length, Src0, VT, MMO, ISD::UNINDEXED,
+      ISD::NON_EXTLOAD, IsExpanding);
+  if (AddToChain)
+    PendingLoads.push_back(Load.getValue(1));
+  setValue(&I, Load);
+}
+
+void SelectionDAGBuilder::visitVariableLengthStore(const CallInst &I) {
+  SDLoc sdl = getCurSDLoc();
+  SDValue Src0 = getValue(I.getArgOperand(0));
+  Value *PtrOperand = I.getArgOperand(1);
+  SDValue Ptr = getValue(PtrOperand);
+  MaybeAlign Alignment(cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
+  SDValue Length = getValue(I.getArgOperand(3));
+  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
+  EVT VT = Src0.getValueType();
+  if (!Alignment)
+    Alignment = DAG.getEVTAlign(VT);
+  AAMDNodes AAInfo;
+  I.getAAMetadata(AAInfo);
+  MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
+      MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
+      VT.getStoreSize().getKnownMinSize(), *Alignment, AAInfo);
+  SDValue StoreNode = DAG.getVariableLengthStore(
+      getMemoryRoot(), sdl, Src0, Ptr, Offset, Length, VT, MMO, ISD::UNINDEXED,
+      false /* IsTruncating */);
+  DAG.setRoot(StoreNode);
+  setValue(&I, StoreNode);
+}
+
 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
                                            bool IsCompressing) {
   SDLoc sdl = getCurSDLoc();
 
   auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
                                MaybeAlign &Alignment) {
     // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
     Src0 = I.getArgOperand(0);
void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::eh_sjlj_longjmp:
     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
                             getRoot(), getValue(I.getArgOperand(0))));
     return;
   case Intrinsic::eh_sjlj_setup_dispatch:
     DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
                             getRoot()));
     return;
+  case Intrinsic::variable_length_load:
+    visitVariableLengthLoad(I);
+    return;
+  case Intrinsic::variable_length_store:
+    visitVariableLengthStore(I);
+    return;
   case Intrinsic::masked_gather:
     visitMaskedGather(I);
     return;
   case Intrinsic::masked_load:
     visitMaskedLoad(I);
     return;
   case Intrinsic::masked_scatter:
     visitMaskedScatter(I);

llvm/lib/CodeGen/TargetPassConfig.cpp

if (getOptLevel() != CodeGenOpt::None && !DisablePartialLibcallInlining)
     addPass(createPartiallyInlineLibCallsPass());
 
   // Instrument function entry and exit, e.g. with calls to mcount().
   addPass(createPostInlineEntryExitInstrumenterPass());
 
   // Add scalarization of target's unsupported masked memory intrinsics pass.
   // the unsupported intrinsic will be replaced with a chain of basic blocks,
   // that stores/loads element one-by-one if the appropriate mask bit is set.
+  addPass(createScalarizeVariableLengthMemIntrinPass());
   addPass(createScalarizeMaskedMemIntrinPass());
 
   // Expand reduction intrinsics into shuffle sequences if the target wants to.
   addPass(createExpandReductionsPass());
 }
 
 /// Turn exception handling constructs into something the code generators can
 /// handle.

llvm/lib/IR/IRBuilder.cpp

assert(Cond->getType() == getInt1Ty() &&
          "an assumption condition must be of type i1");
 
   Value *Ops[] = { Cond };
   Module *M = BB->getParent()->getParent();
   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
   return createCallHelper(FnAssume, Ops, this);
 }
 
+/// Create a call to a Variable-Length Load intrinsic.
+/// \p Ptr       - base pointer for the load
+/// \p Alignment - alignment of the source location
+/// \p Length      - integer which indicates the extent of vector lanes that
+///                should be accessed in memory
+/// \p PassThru  - pass-through value that is used to fill the masked-off lanes
+/// of the result \p Name      - name of the result variable
+CallInst *IRBuilderBase::CreateVariableLengthLoad(Value *Ptr, Align Alignment,
+                                                  Value *Length,
+                                                  Value *PassThru,
+                                                  const Twine &Name) {
+  auto *PtrTy = cast<PointerType>(Ptr->getType());
+  Type *DataTy = PtrTy->getElementType();
+  auto *LengthTy = cast<Type>(Length->getType());
+  assert(DataTy->isVectorTy() && "Ptr should point to a vector");
+  assert(LengthTy->isIntegerTy() && "Length should be an integer");
+  if (!PassThru)
+    PassThru = UndefValue::get(DataTy);
+  Type *OverloadedTypes[] = {DataTy, PtrTy, LengthTy};
+  Value *Ops[] = {Ptr, getInt32(Alignment.value()), Length, PassThru};
+  Module *M = BB->getParent()->getParent();
+  Function *IntrinDecl = Intrinsic::getDeclaration(
+      M, Intrinsic::variable_length_load, OverloadedTypes);
+  return createCallHelper(IntrinDecl, Ops, this, Name);
+}
+
+/// Create a call to a Variable-Length Store intrinsic.
+/// \p Val       - data to be stored,
+/// \p Ptr       - base pointer for the store
+/// \p Alignment - alignment of the destination location
+/// \p Length      - integer which indicates the extent of vector lanes that
+///                should be accessed in memory
+CallInst *IRBuilderBase::CreateVariableLengthStore(Value *Val, Value *Ptr,
+                                                   Align Alignment,
+                                                   Value *Length) {
+  auto *PtrTy = cast<PointerType>(Ptr->getType());
+  Type *DataTy = PtrTy->getElementType();
+  auto *LengthTy = cast<Type>(Length->getType());
+  assert(DataTy->isVectorTy() && "Ptr should point to a vector");
+  assert(LengthTy->isIntegerTy() && "Length should be an integer");
+  Type *OverloadedTypes[] = {DataTy, PtrTy, LengthTy};
+  Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Length};
+  Module *M = BB->getParent()->getParent();
+  Function *IntrinDecl = Intrinsic::getDeclaration(
+      M, Intrinsic::variable_length_store, OverloadedTypes);
+  return createCallHelper(IntrinDecl, Ops, this);
+}
+
 /// Create a call to a Masked Load intrinsic.
 /// \p Ptr       - base pointer for the load
 /// \p Alignment - alignment of the source location
 /// \p Mask      - vector of booleans which indicates what vector lanes should
 ///                be accessed in memory
 /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
 ///                of the result
 /// \p Name      - name of the result variable

llvm/tools/llc/llc.cpp

int main(int argc, char **argv) {
   initializeLoopStrengthReducePass(*Registry);
   initializeLowerIntrinsicsPass(*Registry);
   initializeEntryExitInstrumenterPass(*Registry);
   initializePostInlineEntryExitInstrumenterPass(*Registry);
   initializeUnreachableBlockElimLegacyPassPass(*Registry);
   initializeConstantHoistingLegacyPassPass(*Registry);
   initializeScalarOpts(*Registry);
   initializeVectorization(*Registry);
+  initializeScalarizeVariableLengthMemIntrinPass(*Registry);
   initializeScalarizeMaskedMemIntrinPass(*Registry);
   initializeExpandReductionsPass(*Registry);
   initializeHardwareLoopsPass(*Registry);
   initializeTransformUtils(*Registry);
 
   // Initialize debugging passes.
   initializeScavengerTestPass(*Registry);
 

llvm/tools/opt/opt.cpp

 // it exists.
 static bool IsCodegenPass(StringRef Pass) {
   std::vector<StringRef> PassNamePrefix = {
       "x86-",  "xcore-", "wasm-",    "systemz-", "ppc-",   "nvvm-",   "nvptx-",
       "mips-", "lanai-", "hexagon-", "bpf-",     "avr-",   "thumb2-", "arm-",
       "si-",   "gcn-",   "amdgpu-",  "aarch64-", "amdgcn-"};
   std::vector<StringRef> PassNameContain = {"ehprepare"};
   std::vector<StringRef> PassNameExact = {
-      "safe-stack",           "cost-model",
-      "codegenprepare",       "interleaved-load-combine",
-      "unreachableblockelim", "scalarize-masked-mem-intrin",
-      "verify-safepoint-ir",  "divergence",
-      "infer-address-spaces", "atomic-expand",
-      "hardware-loops",       "type-promotion",
-      "mve-tail-predication", "interleaved-access",
-      "global-merge",         "pre-isel-intrinsic-lowering",
-      "expand-reductions",    "indirectbr-expand",
-      "generic-to-nvvm",      "expandmemcmp"};
+      "safe-stack",
+      "cost-model",
+      "codegenprepare",
+      "interleaved-load-combine",
+      "unreachableblockelim",
+      "scalarize-variable-length-mem-intrin",
+      "scalarize-masked-mem-intrin",
+      "verify-safepoint-ir",
+      "divergence",
+      "infer-address-spaces",
+      "atomic-expand",
+      "hardware-loops",
+      "type-promotion",
+      "mve-tail-predication",
+      "interleaved-access",
+      "global-merge",
+      "pre-isel-intrinsic-lowering",
+      "expand-reductions",
+      "indirectbr-expand",
+      "generic-to-nvvm",
+      "expandmemcmp"};
   for (const auto &P : PassNamePrefix)
     if (Pass.startswith(P))
       return true;
   for (const auto &P : PassNameContain)
     if (Pass.contains(P))
       return true;
   for (const auto &P : PassNameExact)
     if (Pass == P)
int main(int argc, char **argv) {
   initializeTransformUtils(Registry);
   initializeInstCombine(Registry);
   initializeAggressiveInstCombine(Registry);
   initializeInstrumentation(Registry);
   initializeTarget(Registry);
   // For codegen passes, only passes that do IR to IR transformation are
   // supported.
   initializeExpandMemCmpPassPass(Registry);
+  initializeScalarizeVariableLengthMemIntrinPass(Registry);
   initializeScalarizeMaskedMemIntrinPass(Registry);
   initializeCodeGenPreparePass(Registry);
   initializeAtomicExpandPass(Registry);
   initializeRewriteSymbolsLegacyPassPass(Registry);
   initializeWinEHPreparePass(Registry);
   initializeDwarfEHPreparePass(Registry);
   initializeSafeStackLegacyPassPass(Registry);
   initializeSjLjEHPreparePass(Registry);