[VE] Support f128

[kaz7]

Summary

Support f128 using VE instructions. Update regression tests.
I've noticed there is no load or store i128 test, so I add them too.

[simoll]

Please look into the lint comments. Otw, LGTM.

llvm/lib/Target/VE/VEInstrInfo.td

901

unrelated change

[kaz7]

Thanks. I'll update source code following your suggestion and lit results.

llvm/lib/Target/VE/VEInstrInfo.td

901

Thanks. I'll remove this.

[kaz7]

Update code.

llvm/lib/Target/VE/VECallingConv.td

 //===-- VECallingConv.td - Calling Conventions VE ----------*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This describes the calling conventions for the VE architectures.
 //
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // Aurora VE
 //===----------------------------------------------------------------------===//
 def CC_VE_C_Stack: CallingConv<[
+  // F128 are assigned to the stack in 16-byte aligned units
+  CCIfType<[f128], CCAssignToStackWithShadow<16, 16, [SX7]>>,
+
   // All of the rest are assigned to the stack in 8-byte aligned units.
   CCAssignToStack<0, 8>
 ]>;
 
 def CC_VE : CallingConv<[
   // All arguments get passed in generic registers if there is space.
 
   // Promote i1/i8/i16/i32 arguments to i64.
   CCIfType<[i1, i8, i16, i32], CCPromoteToType<i64>>,
 
   // Convert float arguments to i64 with padding.
   //     63     31   0
   //    +------+------+
   //    | float|   0  |
   //    +------+------+
   CCIfType<[f32], CCBitConvertToType<i64>>,
 
   // bool, char, int, enum, long, long long, float, double
   //     --> generic 64 bit registers
   CCIfType<[i64, f64],
            CCAssignToReg<[SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
 
+  // long double --> pair of generic 64 bit registers
+  //
+  // NOTE: If Q1 is allocated while SX1 is free, llvm tries to allocate SX1 for
+  //       following operands, this masks SX1 to avoid such behavior.
+  CCIfType<[f128],
+           CCAssignToRegWithShadow<[Q0, Q1, Q2, Q3],
+                                   [SX0, SX1, SX3, SX5]>>,
+
   // Alternatively, they are assigned to the stack in 8-byte aligned units.
   CCDelegateTo<CC_VE_C_Stack>
 ]>;
 
 // All arguments get passed in stack for varargs function or non-prototyped
 // function.
 def CC_VE2 : CallingConv<[
   // Promote i1/i8/i16/i32 arguments to i64.
   CCIfType<[i1, i8, i16, i32], CCPromoteToType<i64>>,
 
   // Convert float arguments to i64 with padding.
   //     63     31   0
   //    +------+------+
   //    | float|   0  |
   //    +------+------+
   CCIfType<[f32], CCBitConvertToType<i64>>,
 
+  // F128 are assigned to the stack in 16-byte aligned units
+  CCIfType<[f128], CCAssignToStack<16, 16>>,
+
   CCAssignToStack<0, 8>
 ]>;
 
 def RetCC_VE : CallingConv<[
   // Promote i1/i8/i16/i32 return values to i64.
   CCIfType<[i1, i8, i16, i32], CCPromoteToType<i64>>,
 
   // Convert float return values to i64 with padding.
   //     63     31   0
   //    +------+------+
   //    | float|   0  |
   //    +------+------+
   CCIfType<[f32], CCBitConvertToType<i64>>,
 
   // bool, char, int, enum, long, long long, float, double
   //     --> generic 64 bit registers
   CCIfType<[i64, f64],
            CCAssignToReg<[SX0, SX1, SX2, SX3, SX4, SX5, SX6, SX7]>>,
+
+  // long double --> pair of generic 64 bit registers
+  CCIfType<[f128],
+           CCAssignToRegWithShadow<[Q0, Q1, Q2, Q3],
+                                   [SX0, SX1, SX3, SX5]>>,
 ]>;
 
 // Callee-saved registers
 def CSR : CalleeSavedRegs<(add (sequence "SX%u", 18, 33))>;
 def CSR_NoRegs : CalleeSavedRegs<(add)>;
 
 // PreserveAll (clobbers s62,s63) - used for ve_grow_stack
 def CSR_preserve_all : CalleeSavedRegs<(add (sequence "SX%u", 0, 61))>;

llvm/lib/Target/VE/VEISelLowering.h

SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
                       SelectionDAG &DAG) const override;
 
   /// Custom Lower {
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
 
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerToTLSGeneralDynamicModel(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   /// } Custom Lower
 
   /// Custom DAGCombine {
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
   SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const;
   /// } Custom DAGCombine
 
   SDValue withTargetFlags(SDValue Op, unsigned TF, SelectionDAG &DAG) const;
   SDValue makeHiLoPair(SDValue Op, unsigned HiTF, unsigned LoTF,
                        SelectionDAG &DAG) const;
   SDValue makeAddress(SDValue Op, SelectionDAG &DAG) const;
 
+  bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
   /// Returns true if the target allows unaligned memory accesses of the
   /// specified type.
   bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, unsigned Align,
                                       MachineMemOperand::Flags Flags,
                                       bool *Fast) const override;
 

llvm/lib/Target/VE/VEISelLowering.cpp

if (VA.isExtInLoc())
       RV = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), RV);
 
     InVals.push_back(RV);
   }
 
   return Chain;
 }
 
+bool VETargetLowering::isOffsetFoldingLegal(
+    const GlobalAddressSDNode *GA) const {
+  // VE uses 64 bit addressing, so we need multiple instructions to generate
+  // an address.  Folding address with offset increases the number of
+  // instructions, so that we disable it here.  Offsets will be folded in
+  // the DAG combine later if it worth to do so.
+  return false;
+}
+
 /// isFPImmLegal - Returns true if the target can instruction select the
 /// specified FP immediate natively. If false, the legalizer will
 /// materialize the FP immediate as a load from a constant pool.
 bool VETargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                     bool ForCodeSize) const {
   return VT == MVT::f32 || VT == MVT::f64;
 }
 
VETargetLowering::VETargetLowering(const TargetMachine &TM,
   setBooleanContents(ZeroOrOneBooleanContent);
   setBooleanVectorContents(ZeroOrOneBooleanContent);
 
   // Set up the register classes.
   addRegisterClass(MVT::i32, &VE::I32RegClass);
   addRegisterClass(MVT::i64, &VE::I64RegClass);
   addRegisterClass(MVT::f32, &VE::F32RegClass);
   addRegisterClass(MVT::f64, &VE::I64RegClass);
+  addRegisterClass(MVT::f128, &VE::F128RegClass);
 
   /// Load & Store {
   for (MVT FPVT : MVT::fp_valuetypes()) {
     for (MVT OtherFPVT : MVT::fp_valuetypes()) {
       // Turn FP extload into load/fpextend
       setLoadExtAction(ISD::EXTLOAD, FPVT, OtherFPVT, Expand);
 
       // Turn FP truncstore into trunc + store.
VETargetLowering::VETargetLowering(const TargetMachine &TM,
   }
   /// } Load & Store
 
   // Custom legalize address nodes into LO/HI parts.
   MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
   setOperationAction(ISD::BlockAddress, PtrVT, Custom);
   setOperationAction(ISD::GlobalAddress, PtrVT, Custom);
   setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
+  setOperationAction(ISD::ConstantPool, PtrVT, Custom);
 
   /// VAARG handling {
   setOperationAction(ISD::VASTART, MVT::Other, Custom);
   // VAARG needs to be lowered to access with 8 bytes alignment.
   setOperationAction(ISD::VAARG, MVT::Other, Custom);
   // Use the default implementation.
   setOperationAction(ISD::VACOPY, MVT::Other, Expand);
   setOperationAction(ISD::VAEND, MVT::Other, Expand);
VETargetLowering::VETargetLowering(const TargetMachine &TM,
 
   // fp16 not supported
   for (MVT FPVT : MVT::fp_valuetypes()) {
     setOperationAction(ISD::FP16_TO_FP, FPVT, Expand);
     setOperationAction(ISD::FP_TO_FP16, FPVT, Expand);
   }
   /// } Conversion
 
+  /// Floating-point Ops {
+
+  // VE doesn't have fdiv of f128.
+  setOperationAction(ISD::FDIV, MVT::f128, Expand);
+
+  // VE doesn't have load/store of f128, so use custom-lowering.
+  setOperationAction(ISD::LOAD, MVT::f128, Custom);
+  setOperationAction(ISD::STORE, MVT::f128, Custom);
+
+  for (MVT FPVT : {MVT::f32, MVT::f64}) {
+    // f32 and f64 uses ConstantFP.  f128 uses ConstantPool.
+    setOperationAction(ISD::ConstantFP, FPVT, Legal);
+  }
+  /// } Floating-point Ops
+
   setStackPointerRegisterToSaveRestore(VE::SX11);
 
   // We have target-specific dag combine patterns for the following nodes:
   setTargetDAGCombine(ISD::TRUNCATE);
 
   // Set function alignment to 16 bytes
   setMinFunctionAlignment(Align(16));
 
SDValue VETargetLowering::withTargetFlags(SDValue Op, unsigned TF,
   if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
     return DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(GA),
                                       GA->getValueType(0), GA->getOffset(), TF);
 
   if (const BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op))
     return DAG.getTargetBlockAddress(BA->getBlockAddress(), Op.getValueType(),
                                      0, TF);
 
+  if (const ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op))
+    return DAG.getTargetConstantPool(CP->getConstVal(), CP->getValueType(0),
+                                     CP->getAlign(), CP->getOffset(), TF);
+
   if (const ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op))
     return DAG.getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0),
                                        TF);
 
   llvm_unreachable("Unhandled address SDNode");
 }
 
 // Split Op into high and low parts according to HiTF and LoTF.
SDValue VETargetLowering::LowerGlobalAddress(SDValue Op,
   return makeAddress(Op, DAG);
 }
 
 SDValue VETargetLowering::LowerBlockAddress(SDValue Op,
                                             SelectionDAG &DAG) const {
   return makeAddress(Op, DAG);
 }
 
+SDValue VETargetLowering::LowerConstantPool(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  return makeAddress(Op, DAG);
+}
+
 SDValue
 VETargetLowering::LowerToTLSGeneralDynamicModel(SDValue Op,
                                                 SelectionDAG &DAG) const {
   SDLoc dl(Op);
 
   // Generate the following code:
   //   t1: ch,glue = callseq_start t0, 0, 0
   //   t2: i64,ch,glue = VEISD::GETTLSADDR t1, label, t1:1
SDValue VETargetLowering::LowerGlobalTLSAddress(SDValue Op,
   // The current implementation of nld (2.26) doesn't allow local exec model
   // code described in VE-tls_v1.1.pdf (*1) as its input. Instead, we always
   // generate the general dynamic model code sequence.
   //
   // *1: https://www.nec.com/en/global/prod/hpc/aurora/document/VE-tls_v1.1.pdf
   return LowerToTLSGeneralDynamicModel(Op, DAG);
 }
 
+// Lower a f128 load into two f64 loads.
+static SDValue LowerLoadF128(SDValue Op, SelectionDAG &DAG) {
+  SDLoc dl(Op);
+  LoadSDNode *LdNode = dyn_cast<LoadSDNode>(Op.getNode());
+  assert(LdNode && LdNode->getOffset().isUndef() && "Unexpected node type");
+  unsigned alignment = LdNode->getAlign().value();
+  if (alignment > 8)
+    alignment = 8;
+
+  SDValue Lo64 =
+      DAG.getLoad(MVT::f64, dl, LdNode->getChain(), LdNode->getBasePtr(),
+                  LdNode->getPointerInfo(), alignment,
+                  LdNode->isVolatile() ? MachineMemOperand::MOVolatile
+                                       : MachineMemOperand::MONone);
+  EVT addrVT = LdNode->getBasePtr().getValueType();
+  SDValue HiPtr = DAG.getNode(ISD::ADD, dl, addrVT, LdNode->getBasePtr(),
+                              DAG.getConstant(8, dl, addrVT));
+  SDValue Hi64 =
+      DAG.getLoad(MVT::f64, dl, LdNode->getChain(), HiPtr,
+                  LdNode->getPointerInfo(), alignment,
+                  LdNode->isVolatile() ? MachineMemOperand::MOVolatile
+                                       : MachineMemOperand::MONone);
+
+  SDValue SubRegEven = DAG.getTargetConstant(VE::sub_even, dl, MVT::i32);
+  SDValue SubRegOdd = DAG.getTargetConstant(VE::sub_odd, dl, MVT::i32);
+
+  // VE stores Hi64 to 8(addr) and Lo64 to 0(addr)
+  SDNode *InFP128 =
+      DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::f128);
+  InFP128 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, dl, MVT::f128,
+                               SDValue(InFP128, 0), Hi64, SubRegEven);
+  InFP128 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, dl, MVT::f128,
+                               SDValue(InFP128, 0), Lo64, SubRegOdd);
+  SDValue OutChains[2] = {SDValue(Lo64.getNode(), 1),
+                          SDValue(Hi64.getNode(), 1)};
+  SDValue OutChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
+  SDValue Ops[2] = {SDValue(InFP128, 0), OutChain};
+  return DAG.getMergeValues(Ops, dl);
+}
+
+SDValue VETargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
+  LoadSDNode *LdNode = cast<LoadSDNode>(Op.getNode());
+
+  SDValue BasePtr = LdNode->getBasePtr();
+  if (isa<FrameIndexSDNode>(BasePtr.getNode())) {
+    // Do not expand store instruction with frame index here because of
+    // dependency problems.  We expand it later in eliminateFrameIndex().
+    return Op;
+  }
+
+  EVT MemVT = LdNode->getMemoryVT();
+  if (MemVT == MVT::f128)
+    return LowerLoadF128(Op, DAG);
+
+  return Op;
+}
+
+// Lower a f128 store into two f64 stores.
+static SDValue LowerStoreF128(SDValue Op, SelectionDAG &DAG) {
+  SDLoc dl(Op);
+  StoreSDNode *StNode = dyn_cast<StoreSDNode>(Op.getNode());
+  assert(StNode && StNode->getOffset().isUndef() && "Unexpected node type");
+
+  SDValue SubRegEven = DAG.getTargetConstant(VE::sub_even, dl, MVT::i32);
+  SDValue SubRegOdd = DAG.getTargetConstant(VE::sub_odd, dl, MVT::i32);
+
+  SDNode *Hi64 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i64,
+                                    StNode->getValue(), SubRegEven);
+  SDNode *Lo64 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, MVT::i64,
+                                    StNode->getValue(), SubRegOdd);
+
+  unsigned alignment = StNode->getAlign().value();
+  if (alignment > 8)
+    alignment = 8;
+
+  // VE stores Hi64 to 8(addr) and Lo64 to 0(addr)
+  SDValue OutChains[2];
+  OutChains[0] =
+      DAG.getStore(StNode->getChain(), dl, SDValue(Lo64, 0),
+                   StNode->getBasePtr(), MachinePointerInfo(), alignment,
+                   StNode->isVolatile() ? MachineMemOperand::MOVolatile
+                                        : MachineMemOperand::MONone);
+  EVT addrVT = StNode->getBasePtr().getValueType();
+  SDValue HiPtr = DAG.getNode(ISD::ADD, dl, addrVT, StNode->getBasePtr(),
+                              DAG.getConstant(8, dl, addrVT));
+  OutChains[1] =
+      DAG.getStore(StNode->getChain(), dl, SDValue(Hi64, 0), HiPtr,
+                   MachinePointerInfo(), alignment,
+                   StNode->isVolatile() ? MachineMemOperand::MOVolatile
+                                        : MachineMemOperand::MONone);
+  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
+}
+
+SDValue VETargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  StoreSDNode *StNode = cast<StoreSDNode>(Op.getNode());
+  assert(StNode && StNode->getOffset().isUndef() && "Unexpected node type");
+
+  SDValue BasePtr = StNode->getBasePtr();
+  if (isa<FrameIndexSDNode>(BasePtr.getNode())) {
+    // Do not expand store instruction with frame index here because of
+    // dependency problems.  We expand it later in eliminateFrameIndex().
+    return Op;
+  }
+
+  EVT MemVT = StNode->getMemoryVT();
+  if (MemVT == MVT::f128)
+    return LowerStoreF128(Op, DAG);
+
+  // Otherwise, ask llvm to expand it.
+  return SDValue();
+}
+
 SDValue VETargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
   auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Need frame address to find the address of VarArgsFrameIndex.
   MF.getFrameInfo().setFrameAddressIsTaken(true);
 
SDValue VETargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   EVT PtrVT = VAListPtr.getValueType();
   const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
   SDLoc DL(Node);
   SDValue VAList =
       DAG.getLoad(PtrVT, DL, InChain, VAListPtr, MachinePointerInfo(SV));
   SDValue Chain = VAList.getValue(1);
   SDValue NextPtr;
 
-  if (VT == MVT::f32) {
+  if (VT == MVT::f128) {
+    // VE f128 values must be stored with 16 bytes alignment.  We doesn't
+    // know the actual alignment of VAList, so we take alignment of it
+    // dyanmically.
+    int Align = 16;
+    VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
+                         DAG.getConstant(Align - 1, DL, PtrVT));
+    VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
+                         DAG.getConstant(-Align, DL, PtrVT));
+    // Increment the pointer, VAList, by 16 to the next vaarg.
+    NextPtr =
+        DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getIntPtrConstant(16, DL));
+  } else if (VT == MVT::f32) {
     // float --> need special handling like below.
     //    0      4
     //    +------+------+
     //    | empty| float|
     //    +------+------+
     // Increment the pointer, VAList, by 8 to the next vaarg.
     NextPtr =
         DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getIntPtrConstant(8, DL));

 }
 
 SDValue VETargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default:
     llvm_unreachable("Should not custom lower this!");
   case ISD::BlockAddress:
     return LowerBlockAddress(Op, DAG);
+  case ISD::ConstantPool:
+    return LowerConstantPool(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC:
     return lowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::GlobalAddress:
     return LowerGlobalAddress(Op, DAG);
   case ISD::GlobalTLSAddress:
     return LowerGlobalTLSAddress(Op, DAG);
+  case ISD::LOAD:
+    return LowerLOAD(Op, DAG);
+  case ISD::STORE:
+    return LowerSTORE(Op, DAG);
   case ISD::VASTART:
     return LowerVASTART(Op, DAG);
   case ISD::VAARG:
     return LowerVAARG(Op, DAG);
   }
 }
 /// } Custom Lower
 

llvm/lib/Target/VE/VEInstrInfo.cpp

bool VEInstrInfo::reverseBranchCondition(
   return false;
 }
 
 static bool IsAliasOfSX(Register Reg) {
   return VE::I32RegClass.contains(Reg) || VE::I64RegClass.contains(Reg) ||
          VE::F32RegClass.contains(Reg);
 }
 
+static void copyPhysSubRegs(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator I, const DebugLoc &DL,
+                            MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
+                            const MCInstrDesc &MCID, unsigned int numSubRegs,
+                            const unsigned *subRegIdx,
+                            const TargetRegisterInfo *TRI) {
+  MachineInstr *MovMI = nullptr;
+
+  for (unsigned i = 0; i != numSubRegs; ++i) {
+    unsigned SubDest = TRI->getSubReg(DestReg, subRegIdx[i]);
+    unsigned SubSrc = TRI->getSubReg(SrcReg, subRegIdx[i]);
+    assert(SubDest && SubSrc && "Bad sub-register");
+
+    if (MCID.getOpcode() == VE::ORri) {
+      // generate "ORri, dest, src, 0" instruction.
+      MachineInstrBuilder MIB =
+          BuildMI(MBB, I, DL, MCID, SubDest).addReg(SubSrc).addImm(0);
+      MovMI = MIB.getInstr();
+    } else {
+      llvm_unreachable("Unexpected reg-to-reg copy instruction");
+    }
+  }
+  // Add implicit super-register defs and kills to the last MovMI.
+  MovMI->addRegisterDefined(DestReg, TRI);
+  if (KillSrc)
+    MovMI->addRegisterKilled(SrcReg, TRI, true);
+}
+
 void VEInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator I, const DebugLoc &DL,
                               MCRegister DestReg, MCRegister SrcReg,
                               bool KillSrc) const {
 
   if (IsAliasOfSX(SrcReg) && IsAliasOfSX(DestReg)) {
     BuildMI(MBB, I, DL, get(VE::ORri), DestReg)
         .addReg(SrcReg, getKillRegState(KillSrc))
         .addImm(0);
+  } else if (VE::F128RegClass.contains(DestReg, SrcReg)) {
+    // Use two instructions.
+    const unsigned subRegIdx[] = {VE::sub_even, VE::sub_odd};
+    unsigned int numSubRegs = 2;
+    copyPhysSubRegs(MBB, I, DL, DestReg, SrcReg, KillSrc, get(VE::ORri),
+                    numSubRegs, subRegIdx, &getRegisterInfo());
   } else {
     const TargetRegisterInfo *TRI = &getRegisterInfo();
     dbgs() << "Impossible reg-to-reg copy from " << printReg(SrcReg, TRI)
            << " to " << printReg(DestReg, TRI) << "\n";
     llvm_unreachable("Impossible reg-to-reg copy");
   }
 }
 
 /// isLoadFromStackSlot - If the specified machine instruction is a direct
 /// load from a stack slot, return the virtual or physical register number of
 /// the destination along with the FrameIndex of the loaded stack slot.  If
 /// not, return 0.  This predicate must return 0 if the instruction has
 /// any side effects other than loading from the stack slot.
 unsigned VEInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
                                           int &FrameIndex) const {
   if (MI.getOpcode() == VE::LDrii ||    // I64
       MI.getOpcode() == VE::LDLSXrii || // I32
-      MI.getOpcode() == VE::LDUrii      // F32
+      MI.getOpcode() == VE::LDUrii ||   // F32
+      MI.getOpcode() == VE::LDQrii      // F128 (pseudo)
   ) {
     if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
         MI.getOperand(2).getImm() == 0 && MI.getOperand(3).isImm() &&
         MI.getOperand(3).getImm() == 0) {
       FrameIndex = MI.getOperand(1).getIndex();
       return MI.getOperand(0).getReg();
     }
   }
   return 0;
 }
 
 /// isStoreToStackSlot - If the specified machine instruction is a direct
 /// store to a stack slot, return the virtual or physical register number of
 /// the source reg along with the FrameIndex of the loaded stack slot.  If
 /// not, return 0.  This predicate must return 0 if the instruction has
 /// any side effects other than storing to the stack slot.
 unsigned VEInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
                                          int &FrameIndex) const {
   if (MI.getOpcode() == VE::STrii ||  // I64
       MI.getOpcode() == VE::STLrii || // I32
-      MI.getOpcode() == VE::STUrii    // F32
+      MI.getOpcode() == VE::STUrii || // F32
+      MI.getOpcode() == VE::STQrii    // F128 (pseudo)
   ) {
     if (MI.getOperand(0).isFI() && MI.getOperand(1).isImm() &&
         MI.getOperand(1).getImm() == 0 && MI.getOperand(2).isImm() &&
         MI.getOperand(2).getImm() == 0) {
       FrameIndex = MI.getOperand(0).getIndex();
       return MI.getOperand(3).getReg();
     }
   }
BuildMI(MBB, I, DL, get(VE::STLrii))
         .addMemOperand(MMO);
   } else if (RC == &VE::F32RegClass) {
     BuildMI(MBB, I, DL, get(VE::STUrii))
         .addFrameIndex(FI)
         .addImm(0)
         .addImm(0)
         .addReg(SrcReg, getKillRegState(isKill))
         .addMemOperand(MMO);
+  } else if (VE::F128RegClass.hasSubClassEq(RC)) {
+    BuildMI(MBB, I, DL, get(VE::STQrii))
+        .addFrameIndex(FI)
+        .addImm(0)
+        .addImm(0)
+        .addReg(SrcReg, getKillRegState(isKill))
+        .addMemOperand(MMO);
   } else
     report_fatal_error("Can't store this register to stack slot");
 }
 
 void VEInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator I,
                                        Register DestReg, int FI,
                                        const TargetRegisterClass *RC,
BuildMI(MBB, I, DL, get(VE::LDLSXrii), DestReg)
         .addImm(0)
         .addMemOperand(MMO);
   } else if (RC == &VE::F32RegClass) {
     BuildMI(MBB, I, DL, get(VE::LDUrii), DestReg)
         .addFrameIndex(FI)
         .addImm(0)
         .addImm(0)
         .addMemOperand(MMO);
+  } else if (VE::F128RegClass.hasSubClassEq(RC)) {
+    BuildMI(MBB, I, DL, get(VE::LDQrii), DestReg)
+        .addFrameIndex(FI)
+        .addImm(0)
+        .addImm(0)
+        .addMemOperand(MMO);
   } else
     report_fatal_error("Can't load this register from stack slot");
 }
 
 Register VEInstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
   VEMachineFunctionInfo *VEFI = MF->getInfo<VEMachineFunctionInfo>();
   Register GlobalBaseReg = VEFI->getGlobalBaseReg();
   if (GlobalBaseReg != 0)

llvm/lib/Target/VE/VEInstrInfo.td

 // Assembly Language Reference Manual.
 //===----------------------------------------------------------------------===//
 
 //-----------------------------------------------------------------------------
 // Section 8.2 - Load/Store instructions
 //-----------------------------------------------------------------------------
 
 // Multiclass for generic RM instructions
+let hasSideEffects = 0 in
 multiclass RMm<string opcStr, bits<8>opc, RegisterClass RC> {
   def rri : RM<opc, (outs RC:$dest), (ins MEMrri:$addr),
                !strconcat(opcStr, " $dest, $addr"), []>;
   let cy = 0 in
   def rii : RM<opc, (outs RC:$dest), (ins MEMrii:$addr),
                !strconcat(opcStr, " $dest, $addr"), []>;
   let cz = 0 in
   def zri : RM<opc, (outs RC:$dest), (ins MEMzri:$addr),

 defm LD2BZX : LOADm<"ld2b.zx", 0x04, I32, i32, zextloadi16>;
 
 // Section 8.2.6 - LD1B
 let DecoderMethod = "DecodeLoadI32" in
 defm LD1BSX : LOADm<"ld1b.sx", 0x05, I32, i32, sextloadi8>;
 let cx = 1, DecoderMethod = "DecodeLoadI32" in
 defm LD1BZX : LOADm<"ld1b.zx", 0x05, I32, i32, zextloadi8>;
 
+// LDQ pseudo instructions
+let mayLoad = 1, hasSideEffects = 0 in {
+  def LDQrii : Pseudo<(outs F128:$dest), (ins MEMrii:$addr),
+                      "# pseudo ldq $dest, $addr",
+                      [(set f128:$dest, (load ADDRrii:$addr))]>;
+}
+
 // Multiclass for store instructions.
 let mayStore = 1 in
 multiclass STOREm<string opcStr, bits<8> opc, RegisterClass RC, ValueType Ty,
                   SDPatternOperator OpNode = null_frag> {
   def rri : RM<opc, (outs), (ins MEMrri:$addr, RC:$sx),
                !strconcat(opcStr, " $sx, $addr"),
                [(OpNode Ty:$sx, ADDRrri:$addr)]>;
   let cy = 0 in

 // Section 8.2.10 - ST2B
 let DecoderMethod = "DecodeStoreI32" in
 defm ST2B : STOREm<"st2b", 0x14, I32, i32, truncstorei16>;
 
 // Section 8.2.11 - ST1B
 let DecoderMethod = "DecodeStoreI32" in
 defm ST1B : STOREm<"st1b", 0x15, I32, i32, truncstorei8>;
 
+// STQ pseudo instructions
+let mayStore = 1, hasSideEffects = 0 in {
+  def STQrii : Pseudo<(outs), (ins MEMrii:$addr, F128:$sx),
+                      "# pseudo stq $sx, $addr",
+                      [(store f128:$sx, ADDRrii:$addr)]>;
+}
+
 // Section 8.2.12 - DLDS
 let DecoderMethod = "DecodeLoadI64" in
 defm DLD : LOADm<"dld", 0x09, I64, i64, load>;
 
 // Section 8.2.13 - DLDU
 let DecoderMethod = "DecodeLoadF32" in
 defm DLDU : LOADm<"dldu", 0x0a, F32, f32, load>;
 

 let cw = 0, cx = 1 in
 defm FMAXS : RRFm<"fmax.s", 0x3E, F32, f32, fmaxnum, simm7fp, mimmfp32>;
 let cw = 1, cx = 0 in
 defm FMIND : RRFm<"fmin.d", 0x3E, I64, f64, fminnum>;
 let cw = 1, cx = 1 in
 defm FMINS : RRFm<"fmin.s", 0x3E, F32, f32, fminnum, simm7fp, mimmfp32>;
 
 // Section 8.7.7 - FAQ (Floating Add Quadruple)
-defm FADDQ : RRFm<"fadd.q", 0x6C, F128, f128>;
+defm FADDQ : RRFm<"fadd.q", 0x6C, F128, f128, fadd>;
 
 // Section 8.7.8 - FSQ (Floating Subtract Quadruple)
-defm FSUBQ : RRFm<"fsub.q", 0x7C, F128, f128>;
+defm FSUBQ : RRFm<"fsub.q", 0x7C, F128, f128, fsub>;
 
 // Section 8.7.9 - FMQ (Floating Subtract Quadruple)
-defm FMULQ : RRFm<"fmul.q", 0x6D, F128, f128>;
+defm FMULQ : RRFm<"fmul.q", 0x6D, F128, f128, fmul>;
 
 // Section 8.7.10 - FCQ (Floating Compare Quadruple)
 defm FCMPQ : RRNCbm<"fcmp.q", 0x7D, I64, f64, F128, f128, null_frag, simm7fp,
                     mimmfp>;
 
 // Section 8.7.11 - FIX (Convert to Fixed Point)
 // cx: double/float, cw: sx/zx, sz{0-3} = round
 let cx = 0, cw = 0 /* sign extend */ in
multiclass TRUNC64m<SDPatternOperator from,
   def : Pat<(from i64:$src, ADDRzii:$addr),
             (tozii MEMzii:$addr, (EXTRACT_SUBREG $src, sub_i32))>;
 }
 defm : TRUNC64m<truncstorei8, ST1Brri, ST1Brii, ST1Bzri, ST1Bzii>;
 defm : TRUNC64m<truncstorei16, ST2Brri, ST2Brii, ST2Bzri, ST2Bzii>;
 defm : TRUNC64m<truncstorei32, STLrri, STLrii, STLzri, ST1Bzii>;
 
 // Address calculation and its optimization
+def : Pat<(VEhi tconstpool:$in), (LEASLzii 0, 0, tconstpool:$in)>;
+def : Pat<(VElo tconstpool:$in),
+          (ANDrm (LEAzii 0, 0, tconstpool:$in), !add(32, 64))>;
+def : Pat<(add (VEhi tconstpool:$in1), (VElo tconstpool:$in2)),
+          (LEASLrii (ANDrm (LEAzii 0, 0, tconstpool:$in2), !add(32, 64)), 0,
+                    (tconstpool:$in1))>;
+
+// Address calculation and its optimization
 def : Pat<(VEhi tglobaladdr:$in), (LEASLzii 0, 0, tglobaladdr:$in)>;
 def : Pat<(VElo tglobaladdr:$in),
           (ANDrm (LEAzii 0, 0, tglobaladdr:$in), !add(32, 64))>;
 def : Pat<(add (VEhi tglobaladdr:$in1), (VElo tglobaladdr:$in2)),
           (LEASLrii (ANDrm (LEAzii 0, 0, tglobaladdr:$in2), !add(32, 64)), 0,
                     (tglobaladdr:$in1))>;
 
 // GlobalTLS address calculation and its optimization

llvm/lib/Target/VE/VEMCInstLower.cpp

static MCOperand LowerOperand(const MachineInstr *MI, const MachineOperand &MO,
   default:
     report_fatal_error("unsupported operand type");
 
   case MachineOperand::MO_Register:
     if (MO.isImplicit())
       break;
     return MCOperand::createReg(MO.getReg());
 
+  case MachineOperand::MO_ConstantPoolIndex:
+    return LowerSymbolOperand(MI, MO, AP.GetCPISymbol(MO.getIndex()), AP);
   case MachineOperand::MO_ExternalSymbol:
     return LowerSymbolOperand(
         MI, MO, AP.GetExternalSymbolSymbol(MO.getSymbolName()), AP);
   case MachineOperand::MO_GlobalAddress:
     return LowerSymbolOperand(MI, MO, AP.getSymbol(MO.getGlobal()), AP);
   case MachineOperand::MO_Immediate:
     return MCOperand::createImm(MO.getImm());
 

llvm/lib/Target/VE/VERegisterInfo.cpp

void VERegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   const VEFrameLowering *TFI = getFrameLowering(MF);
 
   Register FrameReg;
   int Offset;
   Offset = TFI->getFrameIndexReference(MF, FrameIndex, FrameReg);
 
   Offset += MI.getOperand(FIOperandNum + 2).getImm();
 
+  if (MI.getOpcode() == VE::STQrii) {
+    const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+    unsigned SrcReg = MI.getOperand(3).getReg();
+    unsigned SrcHiReg = getSubReg(SrcReg, VE::sub_even);
+    unsigned SrcLoReg = getSubReg(SrcReg, VE::sub_odd);
+    // VE stores HiReg to 8(addr) and LoReg to 0(addr)
+    MachineInstr *StMI = BuildMI(*MI.getParent(), II, dl, TII.get(VE::STrii))
+                             .addReg(FrameReg)
+                             .addImm(0)
+                             .addImm(0)
+                             .addReg(SrcLoReg);
+    replaceFI(MF, II, *StMI, dl, 0, Offset, FrameReg);
+    MI.setDesc(TII.get(VE::STrii));
+    MI.getOperand(3).setReg(SrcHiReg);
+    Offset += 8;
+  } else if (MI.getOpcode() == VE::LDQrii) {
+    const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+    unsigned DestReg = MI.getOperand(0).getReg();
+    unsigned DestHiReg = getSubReg(DestReg, VE::sub_even);
+    unsigned DestLoReg = getSubReg(DestReg, VE::sub_odd);
+    // VE loads HiReg from 8(addr) and LoReg from 0(addr)
+    MachineInstr *StMI =
+        BuildMI(*MI.getParent(), II, dl, TII.get(VE::LDrii), DestLoReg)
+            .addReg(FrameReg)
+            .addImm(0)
+            .addImm(0);
+    replaceFI(MF, II, *StMI, dl, 1, Offset, FrameReg);
+    MI.setDesc(TII.get(VE::LDrii));
+    MI.getOperand(0).setReg(DestHiReg);
+    Offset += 8;
+  }
+
   replaceFI(MF, II, MI, dl, FIOperandNum, Offset, FrameReg);
 }
 
 Register VERegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   return VE::SX9;
 }
 
 // VE has no architectural need for stack realignment support,

llvm/test/CodeGen/VE/call.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
+declare i32 @sample_add(i32, i32)
+declare i32 @stack_callee_int(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)
+declare i32 @stack_callee_int_szext(i1 signext, i8 zeroext, i32, i32, i32, i32, i32, i32, i16 zeroext, i8 signext)
+declare float @stack_callee_float(float, float, float, float, float, float, float, float, float, float)
+declare void @test(i64)
+
 define i32 @sample_call() {
 ; CHECK-LABEL: sample_call:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s0, sample_add@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, sample_add@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 1, (0)1
 ; CHECK-NEXT:    or %s1, 2, (0)1
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @sample_add(i32 1, i32 2)
   ret i32 %r
 }
 
-declare i32 @sample_add(i32, i32)
-
 define i32 @stack_call_int() {
 ; CHECK-LABEL: stack_call_int:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s0, 10, (0)1
 ; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    or %s34, 9, (0)1
 ; CHECK-NEXT:    lea %s0, stack_callee_int@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, stack_callee_int@hi(, %s0)
 ; CHECK-NEXT:    or %s0, 1, (0)1
 ; CHECK-NEXT:    or %s1, 2, (0)1
 ; CHECK-NEXT:    or %s2, 3, (0)1
 ; CHECK-NEXT:    or %s3, 4, (0)1
 ; CHECK-NEXT:    or %s4, 5, (0)1
 ; CHECK-NEXT:    or %s5, 6, (0)1
 ; CHECK-NEXT:    or %s6, 7, (0)1
 ; CHECK-NEXT:    or %s7, 8, (0)1
 ; CHECK-NEXT:    st %s34, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @stack_callee_int(i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10)
   ret i32 %r
 }
 
-declare i32 @stack_callee_int(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)
-
 define i32 @stack_call_int_szext() {
 ; CHECK-LABEL: stack_call_int_szext:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    or %s0, -1, (0)1
 ; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    lea %s34, 65535
 ; CHECK-NEXT:    lea %s1, stack_callee_int_szext@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s12, stack_callee_int_szext@hi(, %s1)
 ; CHECK-NEXT:    lea %s1, 255
 ; CHECK-NEXT:    or %s2, 3, (0)1
 ; CHECK-NEXT:    or %s3, 4, (0)1
 ; CHECK-NEXT:    or %s4, 5, (0)1
 ; CHECK-NEXT:    or %s5, 6, (0)1
 ; CHECK-NEXT:    or %s6, 7, (0)1
 ; CHECK-NEXT:    or %s7, 8, (0)1
 ; CHECK-NEXT:    st %s34, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call i32 @stack_callee_int_szext(i1 -1, i8 -1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i16 -1, i8 -1)
   ret i32 %r
 }
 
-declare i32 @stack_callee_int_szext(i1 signext, i8 zeroext, i32, i32, i32, i32, i32, i32, i16 zeroext, i8 signext)
-
 define float @stack_call_float() {
 ; CHECK-LABEL: stack_call_float:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s0, 1092616192
 ; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    lea.sl %s34, 1091567616
 ; CHECK-NEXT:    lea.sl %s0, 1065353216
 ; CHECK-NEXT:    lea.sl %s1, 1073741824
 ; CHECK-NEXT:    lea.sl %s2, 1077936128
 ; CHECK-NEXT:    lea.sl %s3, 1082130432
 ; CHECK-NEXT:    lea.sl %s4, 1084227584
 ; CHECK-NEXT:    lea.sl %s5, 1086324736
 ; CHECK-NEXT:    lea.sl %s6, 1088421888
 ; CHECK-NEXT:    lea.sl %s7, 1090519040
 ; CHECK-NEXT:    lea %s35, stack_callee_float@lo
 ; CHECK-NEXT:    and %s35, %s35, (32)0
 ; CHECK-NEXT:    lea.sl %s12, stack_callee_float@hi(, %s35)
 ; CHECK-NEXT:    st %s34, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = tail call float @stack_callee_float(float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0)
   ret float %r
 }
 
-declare float @stack_callee_float(float, float, float, float, float, float, float, float, float, float)
-
 define float @stack_call_float2(float %p0) {
 ; CHECK-LABEL: stack_call_float2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    lea %s1, stack_callee_float@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s12, stack_callee_float@hi(, %s1)
 ; CHECK-NEXT:    st %s0, 240(, %s11)

llvm/test/CodeGen/VE/fp_add.ll

 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fadd.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd double %a, %b
   ret double %r
 }
 
+define fp128 @func3(fp128 %a, fp128 %b) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    fadd.q %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fadd fp128 %a, %b
+  ret fp128 %r
+}
+
 define float @func4(float %a) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 1084227584
 ; CHECK-NEXT:    fadd.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd float %a, 5.000000e+00
   ret float %r
 }
 
 define double @func5(double %a) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 1075052544
 ; CHECK-NEXT:    fadd.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd double %a, 5.000000e+00
   ret double %r
 }
 
+define fp128 @func6(fp128 %a) {
+; CHECK-LABEL: func6:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fadd.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fadd fp128 %a, 0xL00000000000000004001400000000000
+  ret fp128 %r
+}
+
 define float @func7(float %a) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 2139095039
 ; CHECK-NEXT:    fadd.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd float %a, 0x47EFFFFFE0000000
   ret float %r
 }
 
 define double @func8(double %a) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s1, -1
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, 2146435071(, %s1)
 ; CHECK-NEXT:    fadd.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd double %a, 0x7FEFFFFFFFFFFFFF
   ret double %r
 }
 
+define fp128 @func9(fp128 %a) {
+; CHECK-LABEL: func9:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fadd.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fadd fp128 %a, 0xLFFFFFFFFFFFFFFFF7FFEFFFFFFFFFFFF
+  ret fp128 %r
+}
+
 define float @fadds_imm(float %a) {
 ; CHECK-LABEL: fadds_imm:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fadd.s %s0, %s0, (2)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd float %a, -2.e+00
   ret float %r
 }
 
 define double @faddd_imm(double %a) {
 ; CHECK-LABEL: faddd_imm:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fadd.d %s0, %s0, (2)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd double %a, -2.e+00
   ret double %r
 }
+
+define fp128 @faddq_imm(fp128 %a) {
+; CHECK-LABEL: faddq_imm:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fadd.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fadd fp128 %a, 0xLA0000000000000000000000000000000
+  ret fp128 %r
+}

llvm/test/CodeGen/VE/fp_div.ll

 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fdiv.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fdiv double %a, %b
   ret double %r
 }
 
+define fp128 @func3(fp128 %a, fp128 %b) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s4, __divtf3@lo
+; CHECK-NEXT:    and %s4, %s4, (32)0
+; CHECK-NEXT:    lea.sl %s12, __divtf3@hi(, %s4)
+; CHECK-NEXT:    bsic %s10, (, %s12)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fdiv fp128 %a, %b
+  ret fp128 %r
+}
+
 define float @func4(float %a) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 1084227584
 ; CHECK-NEXT:    fdiv.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fdiv float %a, 5.000000e+00
   ret float %r
 }
 
 define double @func5(double %a) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 1075052544
 ; CHECK-NEXT:    fdiv.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fdiv double %a, 5.000000e+00
   ret double %r
 }
 
+define fp128 @func6(fp128 %a) {
+; CHECK-LABEL: func6:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s4, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s2, 8(, %s4)
+; CHECK-NEXT:    ld %s3, (, %s4)
+; CHECK-NEXT:    lea %s4, __divtf3@lo
+; CHECK-NEXT:    and %s4, %s4, (32)0
+; CHECK-NEXT:    lea.sl %s12, __divtf3@hi(, %s4)
+; CHECK-NEXT:    bsic %s10, (, %s12)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fdiv fp128 %a, 0xL00000000000000004001400000000000
+  ret fp128 %r
+}
+
 define float @func7(float %a) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 2139095039
 ; CHECK-NEXT:    fdiv.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fdiv float %a, 0x47EFFFFFE0000000
   ret float %r
 }
 
 define double @func8(double %a) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s1, -1
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, 2146435071(, %s1)
 ; CHECK-NEXT:    fdiv.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fdiv double %a, 0x7FEFFFFFFFFFFFFF
   ret double %r
 }
+
+; Function Attrs: norecurse nounwind readnone
+define fp128 @func9(fp128 %a) {
+; CHECK-LABEL: func9:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s4, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s2, 8(, %s4)
+; CHECK-NEXT:    ld %s3, (, %s4)
+; CHECK-NEXT:    lea %s4, __divtf3@lo
+; CHECK-NEXT:    and %s4, %s4, (32)0
+; CHECK-NEXT:    lea.sl %s12, __divtf3@hi(, %s4)
+; CHECK-NEXT:    bsic %s10, (, %s12)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fdiv fp128 %a, 0xLFFFFFFFFFFFFFFFF7FFEFFFFFFFFFFFF
+  ret fp128 %r
+}

llvm/test/CodeGen/VE/fp_mul.ll

 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fmul.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul double %a, %b
   ret double %r
 }
 
+define fp128 @func3(fp128 %a, fp128 %b) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    fmul.q %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fmul fp128 %a, %b
+  ret fp128 %r
+}
+
 define float @func4(float %a) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 1084227584
 ; CHECK-NEXT:    fmul.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul float %a, 5.000000e+00
   ret float %r
 }
 
 define double @func5(double %a) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 1075052544
 ; CHECK-NEXT:    fmul.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul double %a, 5.000000e+00
   ret double %r
 }
 
+define fp128 @func6(fp128 %a) {
+; CHECK-LABEL: func6:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fmul.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fmul fp128 %a, 0xL00000000000000004001400000000000
+  ret fp128 %r
+}
+
 define float @func7(float %a) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, 2139095039
 ; CHECK-NEXT:    fmul.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul float %a, 0x47EFFFFFE0000000
   ret float %r
 }
 
 define double @func8(double %a) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s1, -1
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, 2146435071(, %s1)
 ; CHECK-NEXT:    fmul.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul double %a, 0x7FEFFFFFFFFFFFFF
   ret double %r
 }
 
+define fp128 @func9(fp128 %a) {
+; CHECK-LABEL: func9:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fmul.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fmul fp128 %a, 0xLFFFFFFFFFFFFFFFF7FFEFFFFFFFFFFFF
+  ret fp128 %r
+}
+
 define float @fmuls_ir(float %a) {
 ; CHECK-LABEL: fmuls_ir:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fmul.s %s0, %s0, (0)1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul float 0.e+00, %a
   ret float %r
 }

 define float @fmuls_ri3(float %a) {
 ; CHECK-LABEL: fmuls_ri3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fmul.s %s0, %s0, (9)0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fmul float %a, 1.175494210692441075487029444849287348827052428745893333857174530571588870475618904265502351336181163787841796875E-38
   ret float %r
 }
-

llvm/test/CodeGen/VE/fp_sub.ll

 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fsub.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fsub double %a, %b
   ret double %r
 }
 
+define fp128 @func3(fp128 %a, fp128 %b) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    fsub.q %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fsub fp128 %a, %b
+  ret fp128 %r
+}
+
 define float @func4(float %a) {
 ; CHECK-LABEL: func4:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, -1063256064
 ; CHECK-NEXT:    fadd.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd float %a, -5.000000e+00
   ret float %r
 }
 
 define double @func5(double %a) {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, -1072431104
 ; CHECK-NEXT:    fadd.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd double %a, -5.000000e+00
   ret double %r
 }
 
+define fp128 @func6(fp128 %a) {
+; CHECK-LABEL: func6:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fadd.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fadd fp128 %a, 0xL0000000000000000C001400000000000
+  ret fp128 %r
+}
+
 define float @func7(float %a) {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea.sl %s1, -8388609
 ; CHECK-NEXT:    fadd.s %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd float %a, 0xC7EFFFFFE0000000
   ret float %r
 }
 
 define double @func8(double %a) {
 ; CHECK-LABEL: func8:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s1, -1
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, -1048577(, %s1)
 ; CHECK-NEXT:    fadd.d %s0, %s0, %s1
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fadd double %a, 0xFFEFFFFFFFFFFFFF
   ret double %r
 }
 
+define fp128 @func9(fp128 %a) {
+; CHECK-LABEL: func9:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
+; CHECK-NEXT:    ld %s4, 8(, %s2)
+; CHECK-NEXT:    ld %s5, (, %s2)
+; CHECK-NEXT:    fadd.q %s0, %s0, %s4
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = fadd fp128 %a, 0xLFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFF
+  ret fp128 %r
+}
+
 define float @fsubs_ir(float %a) {
 ; CHECK-LABEL: fsubs_ir:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    fsub.s %s0, 0, %s0
 ; CHECK-NEXT:    or %s11, 0, %s9
   %r = fsub float 0.e+00, %a
   ret float %r
 }

llvm/test/CodeGen/VE/load.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 ; Function Attrs: norecurse nounwind readonly
+define fp128 @loadf128(fp128* nocapture readonly %0) {
+; CHECK-LABEL: loadf128:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    ld %s2, 8(, %s0)
+; CHECK-NEXT:    ld %s3, (, %s0)
+; CHECK-NEXT:    or %s0, 0, %s2
+; CHECK-NEXT:    or %s1, 0, %s3
+; CHECK-NEXT:    or %s11, 0, %s9
+  %2 = load fp128, fp128* %0, align 16
+  ret fp128 %2
+}
+
+; Function Attrs: norecurse nounwind readonly
 define double @loadf64(double* nocapture readonly %0) {
 ; CHECK-LABEL: loadf64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = load double, double* %0, align 16
   ret double %2
 }
 
 ; Function Attrs: norecurse nounwind readonly
 define float @loadf32(float* nocapture readonly %0) {
 ; CHECK-LABEL: loadf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ldu %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = load float, float* %0, align 16
   ret float %2
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define i128 @loadi128(i128* nocapture readonly %0) {
+; CHECK-LABEL: loadi128:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    ld %s2, (, %s0)
+; CHECK-NEXT:    ld %s1, 8(, %s0)
+; CHECK-NEXT:    or %s0, 0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %2 = load i128, i128* %0, align 16
+  ret i128 %2
+}
+
+; Function Attrs: norecurse nounwind readonly
 define i64 @loadi64(i64* nocapture readonly %0) {
 ; CHECK-LABEL: loadi64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = load i64, i64* %0, align 16
   ret i64 %2
 }

 ; CHECK-NEXT:    ld1b.zx %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %2 = load i8, i8* %0, align 16
   %3 = zext i8 %2 to i64
   ret i64 %3
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define fp128 @loadf128stk() {
+; CHECK-LABEL: loadf128stk:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    ld %s1, 176(, %s11)
+; CHECK-NEXT:    ld %s0, 184(, %s11)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %addr = alloca fp128, align 16
+  %1 = load fp128, fp128* %addr, align 16
+  ret fp128 %1
+}
+
+; Function Attrs: norecurse nounwind readonly
 define double @loadf64stk() {
 ; CHECK-LABEL: loadf64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld %s0, 176(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %addr = alloca double, align 16
   %1 = load double, double* %addr, align 16
   ret double %1
 }
 
 ; Function Attrs: norecurse nounwind readonly
 define float @loadf32stk() {
 ; CHECK-LABEL: loadf32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ldu %s0, 176(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %addr = alloca float, align 16
   %1 = load float, float* %addr, align 16
   ret float %1
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define i128 @loadi128stk() {
+; CHECK-LABEL: loadi128stk:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    ld %s0, 176(, %s11)
+; CHECK-NEXT:    ld %s1, 184(, %s11)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %addr = alloca i128, align 16
+  %1 = load i128, i128* %addr, align 16
+  ret i128 %1
+}
+
+; Function Attrs: norecurse nounwind readonly
 define i64 @loadi64stk() {
 ; CHECK-LABEL: loadi64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    ld %s0, 176(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %addr = alloca i64, align 16
   %1 = load i64, i64* %addr, align 16
   ret i64 %1

llvm/test/CodeGen/VE/load_gv.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 @vi8 = common dso_local local_unnamed_addr global i8 0, align 1
 @vi16 = common dso_local local_unnamed_addr global i16 0, align 2
 @vi32 = common dso_local local_unnamed_addr global i32 0, align 4
 @vi64 = common dso_local local_unnamed_addr global i64 0, align 8
+@vi128 = common dso_local local_unnamed_addr global i128 0, align 16
 @vf32 = common dso_local local_unnamed_addr global float 0.000000e+00, align 4
 @vf64 = common dso_local local_unnamed_addr global double 0.000000e+00, align 8
+@vf128 = common dso_local local_unnamed_addr global fp128 0xL00000000000000000000000000000000, align 16
+
+; Function Attrs: norecurse nounwind readonly
+define fp128 @loadf128com() {
+; CHECK-LABEL: loadf128com:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, vf128@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s2, vf128@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 8(, %s2)
+; CHECK-NEXT:    ld %s1, (, %s2)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %1 = load fp128, fp128* @vf128, align 16
+  ret fp128 %1
+}
 
 ; Function Attrs: norecurse nounwind readonly
 define double @loadf64com() {
 ; CHECK-LABEL: loadf64com:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s0, vf64@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, vf64@hi(, %s0)

 ; CHECK-NEXT:    lea.sl %s0, vf32@hi(, %s0)
 ; CHECK-NEXT:    ldu %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %1 = load float, float* @vf32, align 4
   ret float %1
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define i128 @loadi128com() {
+; CHECK-LABEL: loadi128com:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s0, vi128@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s1, vi128@hi(, %s0)
+; CHECK-NEXT:    ld %s0, (, %s1)
+; CHECK-NEXT:    ld %s1, 8(, %s1)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %1 = load i128, i128* @vi128, align 16
+  ret i128 %1
+}
+
+; Function Attrs: norecurse nounwind readonly
 define i64 @loadi64com() {
 ; CHECK-LABEL: loadi64com:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s0, vi64@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, vi64@hi(, %s0)
 ; CHECK-NEXT:    ld %s0, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9

llvm/test/CodeGen/VE/load_off.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 @bufi8 = global [3 x i8] zeroinitializer, align 1
 @bufi16 = global [3 x i16] zeroinitializer, align 2
 @bufi32 = global [3 x i32] zeroinitializer, align 4
 @bufi64 = global [3 x i64] zeroinitializer, align 8
 @bufi128 = global [3 x i128] zeroinitializer, align 16
 @buff32 = global [3 x float] zeroinitializer, align 4
 @buff64 = global [3 x double] zeroinitializer, align 8
 @buff128 = global [3 x fp128] zeroinitializer, align 16
 
 ; Function Attrs: noinline nounwind optnone
 define signext i8 @loadi8s() {
 ; CHECK-LABEL: loadi8s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi8+2@lo
+; CHECK-NEXT:    lea %s0, bufi8@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi8+2@hi(, %s0)
-; CHECK-NEXT:    ld1b.sx %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi8@hi(, %s0)
+; CHECK-NEXT:    ld1b.sx %s0, 2(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i8, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @bufi8, i64 0, i64 2), align 1
   ret i8 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define signext i16 @loadi16s() {
 ; CHECK-LABEL: loadi16s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi16+4@lo
+; CHECK-NEXT:    lea %s0, bufi16@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi16+4@hi(, %s0)
-; CHECK-NEXT:    ld2b.sx %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi16@hi(, %s0)
+; CHECK-NEXT:    ld2b.sx %s0, 4(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* @bufi16, i64 0, i64 2), align 2
   ret i16 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define signext i32 @loadi32s() {
 ; CHECK-LABEL: loadi32s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi32+8@lo
+; CHECK-NEXT:    lea %s0, bufi32@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi32+8@hi(, %s0)
-; CHECK-NEXT:    ldl.sx %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi32@hi(, %s0)
+; CHECK-NEXT:    ldl.sx %s0, 8(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i32, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @bufi32, i64 0, i64 2), align 4
   ret i32 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define i64 @loadi64s() {
 ; CHECK-LABEL: loadi64s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi64+16@lo
+; CHECK-NEXT:    lea %s0, bufi64@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi64+16@hi(, %s0)
-; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi64@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 16(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @bufi64, i64 0, i64 2), align 8
   ret i64 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define i128 @loadi128s() {
 ; CHECK-LABEL: loadi128s:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi128+32@lo
+; CHECK-NEXT:    lea %s0, bufi128@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi128+32@hi(, %s0)
-; CHECK-NEXT:    ld %s0, (, %s0)
-; CHECK-NEXT:    lea %s1, bufi128+40@lo
-; CHECK-NEXT:    and %s1, %s1, (32)0
-; CHECK-NEXT:    lea.sl %s1, bufi128+40@hi(, %s1)
-; CHECK-NEXT:    ld %s1, (, %s1)
+; CHECK-NEXT:    lea.sl %s1, bufi128@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 32(, %s1)
+; CHECK-NEXT:    ld %s1, 40(, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i128, i128* getelementptr inbounds ([3 x i128], [3 x i128]* @bufi128, i64 0, i64 2), align 16
   ret i128 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define zeroext i8 @loadi8z() {
 ; CHECK-LABEL: loadi8z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi8+2@lo
+; CHECK-NEXT:    lea %s0, bufi8@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi8+2@hi(, %s0)
-; CHECK-NEXT:    ld1b.zx %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi8@hi(, %s0)
+; CHECK-NEXT:    ld1b.zx %s0, 2(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i8, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @bufi8, i64 0, i64 2), align 1
   ret i8 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define zeroext i16 @loadi16z() {
 ; CHECK-LABEL: loadi16z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi16+4@lo
+; CHECK-NEXT:    lea %s0, bufi16@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi16+4@hi(, %s0)
-; CHECK-NEXT:    ld2b.zx %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi16@hi(, %s0)
+; CHECK-NEXT:    ld2b.zx %s0, 4(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i16, i16* getelementptr inbounds ([3 x i16], [3 x i16]* @bufi16, i64 0, i64 2), align 2
   ret i16 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define zeroext i32 @loadi32z() {
 ; CHECK-LABEL: loadi32z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi32+8@lo
+; CHECK-NEXT:    lea %s0, bufi32@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi32+8@hi(, %s0)
-; CHECK-NEXT:    ldl.zx %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi32@hi(, %s0)
+; CHECK-NEXT:    ldl.zx %s0, 8(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i32, i32* getelementptr inbounds ([3 x i32], [3 x i32]* @bufi32, i64 0, i64 2), align 4
   ret i32 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define i64 @loadi64z() {
 ; CHECK-LABEL: loadi64z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi64+16@lo
+; CHECK-NEXT:    lea %s0, bufi64@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi64+16@hi(, %s0)
-; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, bufi64@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 16(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i64, i64* getelementptr inbounds ([3 x i64], [3 x i64]* @bufi64, i64 0, i64 2), align 8
   ret i64 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define i128 @loadi128z() {
 ; CHECK-LABEL: loadi128z:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, bufi128+32@lo
+; CHECK-NEXT:    lea %s0, bufi128@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, bufi128+32@hi(, %s0)
-; CHECK-NEXT:    ld %s0, (, %s0)
-; CHECK-NEXT:    lea %s1, bufi128+40@lo
-; CHECK-NEXT:    and %s1, %s1, (32)0
-; CHECK-NEXT:    lea.sl %s1, bufi128+40@hi(, %s1)
-; CHECK-NEXT:    ld %s1, (, %s1)
+; CHECK-NEXT:    lea.sl %s1, bufi128@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 32(, %s1)
+; CHECK-NEXT:    ld %s1, 40(, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load i128, i128* getelementptr inbounds ([3 x i128], [3 x i128]* @bufi128, i64 0, i64 2), align 16
   ret i128 %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define float @loadf32() {
 ; CHECK-LABEL: loadf32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, buff32+8@lo
+; CHECK-NEXT:    lea %s0, buff32@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, buff32+8@hi(, %s0)
-; CHECK-NEXT:    ldu %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, buff32@hi(, %s0)
+; CHECK-NEXT:    ldu %s0, 8(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load float, float* getelementptr inbounds ([3 x float], [3 x float]* @buff32, i64 0, i64 2), align 4
   ret float %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define double @loadf64() {
 ; CHECK-LABEL: loadf64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, buff64+16@lo
+; CHECK-NEXT:    lea %s0, buff64@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, buff64+16@hi(, %s0)
-; CHECK-NEXT:    ld %s0, (, %s0)
+; CHECK-NEXT:    lea.sl %s0, buff64@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 16(, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load double, double* getelementptr inbounds ([3 x double], [3 x double]* @buff64, i64 0, i64 2), align 8
   ret double %0
 }
 
 ; Function Attrs: noinline nounwind optnone
 define fp128 @loadf128() {
 ; CHECK-LABEL: loadf128:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    lea %s0, buff128+32@lo
+; CHECK-NEXT:    lea %s0, buff128@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
-; CHECK-NEXT:    lea.sl %s0, buff128+32@hi(, %s0)
-; CHECK-NEXT:    ld %s0, (, %s0)
-; CHECK-NEXT:    lea %s1, buff128+40@lo
-; CHECK-NEXT:    and %s1, %s1, (32)0
-; CHECK-NEXT:    lea.sl %s1, buff128+40@hi(, %s1)
-; CHECK-NEXT:    ld %s1, (, %s1)
+; CHECK-NEXT:    lea.sl %s2, buff128@hi(, %s0)
+; CHECK-NEXT:    ld %s0, 40(, %s2)
+; CHECK-NEXT:    ld %s1, 32(, %s2)
 ; CHECK-NEXT:    or %s11, 0, %s9
 entry:
   %0 = load fp128, fp128* getelementptr inbounds ([3 x fp128], [3 x fp128]* @buff128, i64 0, i64 2), align 16
   ret fp128 %0
 }

llvm/test/CodeGen/VE/store.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 ; Function Attrs: norecurse nounwind readonly
+define void @storef128(fp128* nocapture %0, fp128 %1) {
+; CHECK-LABEL: storef128:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    st %s2, 8(, %s0)
+; CHECK-NEXT:    st %s3, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+  store fp128 %1, fp128* %0, align 16
+  ret void
+}
+
+; Function Attrs: norecurse nounwind readonly
 define void @storef64(double* nocapture %0, double %1) {
 ; CHECK-LABEL: storef64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s1, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   store double %1, double* %0, align 16
   ret void
 }
 
 ; Function Attrs: norecurse nounwind readonly
 define void @storef32(float* nocapture %0, float %1) {
 ; CHECK-LABEL: storef32:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    stu %s1, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   store float %1, float* %0, align 16
   ret void
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define void @storei128(i128* nocapture %0, i128 %1) {
+; CHECK-LABEL: storei128:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    st %s2, 8(, %s0)
+; CHECK-NEXT:    st %s1, (, %s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+  store i128 %1, i128* %0, align 16
+  ret void
+}
+
+; Function Attrs: norecurse nounwind readonly
 define void @storei64(i64* nocapture %0, i64 %1) {
 ; CHECK-LABEL: storei64:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s1, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   store i64 %1, i64* %0, align 16
   ret void
 }

 ; CHECK-NEXT:    st1b %s1, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %3 = trunc i64 %1 to i8
   store i8 %3, i8* %0, align 16
   ret void
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define void @storef128stk(fp128 %0) {
+; CHECK-LABEL: storef128stk:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    st %s1, 176(, %s11)
+; CHECK-NEXT:    st %s0, 184(, %s11)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %addr = alloca fp128, align 16
+  store fp128 %0, fp128* %addr, align 16
+  ret void
+}
+
+; Function Attrs: norecurse nounwind readonly
 define void @storef64stk(double %0) {
 ; CHECK-LABEL: storef64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s0, 176(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %addr = alloca double, align 16
   store double %0, double* %addr, align 16
   ret void
 }
 
 ; Function Attrs: norecurse nounwind readonly
 define void @storef32stk(float %0) {
 ; CHECK-LABEL: storef32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    stu %s0, 176(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %addr = alloca float, align 16
   store float %0, float* %addr, align 16
   ret void
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define void @storei128stk(i128 %0) {
+; CHECK-LABEL: storei128stk:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    st %s1, 184(, %s11)
+; CHECK-NEXT:    st %s0, 176(, %s11)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %addr = alloca i128, align 16
+  store i128 %0, i128* %addr, align 16
+  ret void
+}
+
+; Function Attrs: norecurse nounwind readonly
 define void @storei64stk(i64 %0) {
 ; CHECK-LABEL: storei64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s0, 176(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
   %addr = alloca i64, align 16
   store i64 %0, i64* %addr, align 16
   ret void

llvm/test/CodeGen/VE/store_gv.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 @vi8 = common dso_local local_unnamed_addr global i8 0, align 1
 @vi16 = common dso_local local_unnamed_addr global i16 0, align 2
 @vi32 = common dso_local local_unnamed_addr global i32 0, align 4
 @vi64 = common dso_local local_unnamed_addr global i64 0, align 8
+@vi128 = common dso_local local_unnamed_addr global i128 0, align 16
 @vf32 = common dso_local local_unnamed_addr global float 0.000000e+00, align 4
 @vf64 = common dso_local local_unnamed_addr global double 0.000000e+00, align 8
+@vf128 = common dso_local local_unnamed_addr global fp128 0xL00000000000000000000000000000000, align 16
+
+; Function Attrs: norecurse nounwind readonly
+define void @storef128com(fp128 %0) {
+; CHECK-LABEL: storef128com:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, vf128@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, vf128@hi(, %s2)
+; CHECK-NEXT:    st %s0, 8(, %s2)
+; CHECK-NEXT:    st %s1, (, %s2)
+; CHECK-NEXT:    or %s11, 0, %s9
+  store fp128 %0, fp128* @vf128, align 16
+  ret void
+}
 
 ; Function Attrs: norecurse nounwind readonly
 define void @storef64com(double %0) {
 ; CHECK-LABEL: storef64com:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s1, vf64@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, vf64@hi(, %s1)

 ; CHECK-NEXT:    lea.sl %s1, vf32@hi(, %s1)
 ; CHECK-NEXT:    stu %s0, (, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9
   store float %0, float* @vf32, align 4
   ret void
 }
 
 ; Function Attrs: norecurse nounwind readonly
+define void @storei128com(i128 %0) {
+; CHECK-LABEL: storei128com:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s2, vi128@lo
+; CHECK-NEXT:    and %s2, %s2, (32)0
+; CHECK-NEXT:    lea.sl %s2, vi128@hi(, %s2)
+; CHECK-NEXT:    st %s1, 8(, %s2)
+; CHECK-NEXT:    st %s0, (, %s2)
+; CHECK-NEXT:    or %s11, 0, %s9
+  store i128 %0, i128* @vi128, align 16
+  ret void
+}
+
+; Function Attrs: norecurse nounwind readonly
 define void @storei64com(i64 %0) {
 ; CHECK-LABEL: storei64com:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    lea %s1, vi64@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, vi64@hi(, %s1)
 ; CHECK-NEXT:    st %s0, (, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9

 ; CHECK-NEXT:    lea %s1, vi8@lo
 ; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lea.sl %s1, vi8@hi(, %s1)
 ; CHECK-NEXT:    st1b %s0, (, %s1)
 ; CHECK-NEXT:    or %s11, 0, %s9
   store i8 %0, i8* @vi8, align 1
   ret void
 }
-

llvm/test/CodeGen/VE/va_arg.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 @.str = private unnamed_addr constant [6 x i8] c"a=%d\0A\00", align 1
 @.str.1 = private unnamed_addr constant [6 x i8] c"b=%d\0A\00", align 1
 @.str.2 = private unnamed_addr constant [6 x i8] c"c=%d\0A\00", align 1
 @.str.3 = private unnamed_addr constant [6 x i8] c"d=%u\0A\00", align 1
 @.str.4 = private unnamed_addr constant [6 x i8] c"e=%u\0A\00", align 1
 @.str.5 = private unnamed_addr constant [6 x i8] c"f=%u\0A\00", align 1
 @.str.6 = private unnamed_addr constant [6 x i8] c"g=%f\0A\00", align 1
 @.str.7 = private unnamed_addr constant [6 x i8] c"h=%p\0A\00", align 1
 @.str.8 = private unnamed_addr constant [7 x i8] c"i=%ld\0A\00", align 1
 @.str.9 = private unnamed_addr constant [7 x i8] c"j=%lf\0A\00", align 1
+@.str.10 = private unnamed_addr constant [7 x i8] c"j=%Lf\0A\00", align 1
 
 define i32 @func_vainout(i32, ...) {
 ; CHECK-LABEL: func_vainout:
 ; CHECK:         ldl.sx %s1, 184(, %s9)
-; CHECK:         ld2b.sx %s18, 192(, %s9)
-; CHECK:         ld1b.sx %s19, 200(, %s9)
-; CHECK:         ldl.sx %s20, 208(, %s9)
-; CHECK:         ld2b.zx %s21, 216(, %s9)
-; CHECK:         ld1b.zx %s22, 224(, %s9)
-; CHECK:         ldu %s23, 236(, %s9)
-; CHECK:         ld %s24, 240(, %s9)
-; CHECK:         ld %s25, 248(, %s9)
-; CHECK:         ld %s26, 256(, %s9)
+; CHECK:         ld2b.sx %s19, 192(, %s9)
+; CHECK:         ld1b.sx %s22, 200(, %s9)
+; CHECK:         ldl.sx %s23, 208(, %s9)
+; CHECK:         ld2b.zx %s24, 216(, %s9)
+; CHECK:         ld1b.zx %s25, 224(, %s9)
+; CHECK:         ldu %s26, 236(, %s9)
+; CHECK:         ld %s27, 240(, %s9)
+; CHECK:         ld %s28, 248(, %s9)
+; CHECK:         ld %s29, 256(, %s9)
+; CHECK:         lea %s0, 279(, %s9)
+; CHECK:         and %s0, -16, %s0
+; CHECK:         lea %s2, 16(, %s0)
+; CHECK:         ld %s20, 8(, %s0)
+; CHECK:         ld %s21, (, %s0)
+; CHECK:         ld %s18, 16(, %s0)
 
   %a = alloca i8*, align 8
   %a8 = bitcast i8** %a to i8*
   call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %a8)
   call void @llvm.va_start(i8* nonnull %a8)
   %p0 = va_arg i8** %a, i32
   %p1 = va_arg i8** %a, i16
   %p2 = va_arg i8** %a, i8
   %p3 = va_arg i8** %a, i32
   %p4 = va_arg i8** %a, i16
   %p5 = va_arg i8** %a, i8
   %p6 = va_arg i8** %a, float
   %p7 = va_arg i8** %a, i8*
   %p8 = va_arg i8** %a, i64
   %p9 = va_arg i8** %a, double
+  %p10 = va_arg i8** %a, fp128
+  %p11 = va_arg i8** %a, double
   call void @llvm.va_end(i8* nonnull %a8)
   call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %a8)
   %pf0 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0), i32 %p0)
   %p1.s32 = sext i16 %p1 to i32
   %pf1 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.1, i64 0, i64 0), i32 %p1.s32)
   %p2.s32 = sext i8 %p2 to i32
   %pf2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.2, i64 0, i64 0), i32 %p2.s32)
   %pf3 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.3, i64 0, i64 0), i32 %p3)
   %p4.z32 = zext i16 %p4 to i32
   %pf4 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.4, i64 0, i64 0), i32 %p4.z32)
   %p5.z32 = zext i8 %p5 to i32
   %pf5 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.5, i64 0, i64 0), i32 %p5.z32)
   %pf6 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.6, i64 0, i64 0), float %p6)
   %pf7 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str.7, i64 0, i64 0), i8* %p7)
   %pf8 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str.8, i64 0, i64 0), i64 %p8)
   %pf9 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str.9, i64 0, i64 0), double %p9)
+  %pf10 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str.10, i64 0, i64 0), fp128 %p10)
+  %pf11 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str.9, i64 0, i64 0), double %p11)
   ret i32 0
 }
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
 declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 declare void @llvm.va_start(i8*)
 declare void @llvm.va_end(i8*)
 declare i32 @printf(i8* nocapture readonly, ...)

llvm/test/CodeGen/VE/va_caller.ll

 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 declare i32 @func(i32, ...)
 
 define i32 @caller() {
 ; CHECK-LABEL: caller:
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    st %s18, 48(, %s9) # 8-byte Folded Spill
 ; CHECK-NEXT:    or %s18, 0, (0)1
-; CHECK-NEXT:    st %s18, 280(, %s11)
-; CHECK-NEXT:    or %s0, 11, (0)1
-; CHECK-NEXT:    st %s0, 272(, %s11)
 ; CHECK-NEXT:    st %s18, 264(, %s11)
 ; CHECK-NEXT:    or %s0, 10, (0)1
 ; CHECK-NEXT:    st %s0, 256(, %s11)
 ; CHECK-NEXT:    lea.sl %s0, 1075970048
 ; CHECK-NEXT:    st %s0, 248(, %s11)
 ; CHECK-NEXT:    or %s0, 8, (0)1
 ; CHECK-NEXT:    st %s0, 240(, %s11)
 ; CHECK-NEXT:    st %s18, 232(, %s11)
 ; CHECK-NEXT:    or %s5, 5, (0)1
 ; CHECK-NEXT:    st %s5, 216(, %s11)
 ; CHECK-NEXT:    or %s4, 4, (0)1
 ; CHECK-NEXT:    st %s4, 208(, %s11)
 ; CHECK-NEXT:    or %s3, 3, (0)1
 ; CHECK-NEXT:    st %s3, 200(, %s11)
 ; CHECK-NEXT:    or %s2, 2, (0)1
 ; CHECK-NEXT:    st %s2, 192(, %s11)
 ; CHECK-NEXT:    or %s1, 1, (0)1
 ; CHECK-NEXT:    st %s1, 184(, %s11)
+; CHECK-NEXT:    lea %s0, .LCPI{{[0-9]+}}_0@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s0, .LCPI{{[0-9]+}}_0@hi(, %s0)
+; CHECK-NEXT:    ld %s34, 8(, %s0)
+; CHECK-NEXT:    ld %s35, (, %s0)
 ; CHECK-NEXT:    st %s18, 176(, %s11)
 ; CHECK-NEXT:    lea.sl %s6, 1086324736
+; CHECK-NEXT:    st %s6, 224(, %s11)
+; CHECK-NEXT:    st %s34, 280(, %s11)
 ; CHECK-NEXT:    lea %s0, func@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s12, func@hi(, %s0)
-; CHECK-NEXT:    st %s6, 224(, %s11)
+; CHECK-NEXT:    st %s35, 272(, %s11)
 ; CHECK-NEXT:    or %s0, 0, %s18
 ; CHECK-NEXT:    or %s7, 0, %s18
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s0, 0, %s18
 ; CHECK-NEXT:    ld %s18, 48(, %s9) # 8-byte Folded Reload
 ; CHECK-NEXT:    or %s11, 0, %s9
-  call i32 (i32, ...) @func(i32 0, i16 1, i8 2, i32 3, i16 4, i8 5, float 6.0, i8* null, i64 8, double 9.0, i128 10, i128 11)
+  call i32 (i32, ...) @func(i32 0, i16 1, i8 2, i32 3, i16 4, i8 5, float 6.0, i8* null, i64 8, double 9.0, i128 10, fp128 0xLA000000000000000)
   ret i32 0
 }