[RISCV] Move creation of constant pools from isel to lowering.

[craig.topper]

Summary

This simplifies the isel code by removing the manual load creation.
It also improves our ability to use 0 strided loads for vector splats.

There is an assumption here that Mask and ShiftedMask constants are
cheap enough that they don't become constant pool loads so that our
isel optimizations involving And still work. I believe those constants
are 3 instructions in the worst case.

The rv64zbp-intrinsic.ll changes is a regression caused by intrinsics
being expanded to RISCVISD also occuring during lowering. So the optimizations
were only happening during the last DAGCombine, which can't see through the
load. I believe we can fix this test by implementing
TargetLowering::getTargetConstantFromLoad for RISC-V or by adding the intrinsic
to computeKnownBitsForTargetNode to enable earlier DAG combine. Since Zbp is not
a ratified extension, I don't view these as blocking this patch.

[craig.topper]

Move getAddr back to being a private member of RISCVTargetLowering

[wangpc]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

3642

I think these instantiations can be removed now.

[craig.topper]

Remove unneeded temporary instantiations

[reames]

LGTM

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

2973

Minor: This check should be redundant except maybe as a minor compile time win assuming the count threshold just below is at least 2.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

2973

Agreed, I mainly did it for the compile time win. getMaxBuildIntsCost has a max with 2 in it.

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
case RISCV::FSD:
     BaseOpIdx = 1;
     OffsetOpIdx = 2;
     return true;
   }
 
   return false;
 }
 
-static SDNode *selectImmWithConstantPool(SelectionDAG *CurDAG, const SDLoc &DL,
-                                         const MVT VT, int64_t Imm,
-                                         const RISCVSubtarget &Subtarget) {
-  assert(VT == MVT::i64 && "Expecting MVT::i64");
-  const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
-  ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(CurDAG->getConstantPool(
-      ConstantInt::get(EVT(VT).getTypeForEVT(*CurDAG->getContext()), Imm), VT));
-  SDValue Addr = TLI->getAddr(CP, *CurDAG);
-  SDValue Offset = CurDAG->getTargetConstant(0, DL, VT);
-  // Since there is no data race, the chain can be the entry node.
-  SDNode *Load = CurDAG->getMachineNode(RISCV::LD, DL, VT, Addr, Offset,
-                                        CurDAG->getEntryNode());
-  MachineFunction &MF = CurDAG->getMachineFunction();
-  MachineMemOperand *MemOp = MF.getMachineMemOperand(
-      MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
-      LLT(VT), CP->getAlign());
-  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Load), {MemOp});
-  return Load;
-}
-
 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
                          int64_t Imm, const RISCVSubtarget &Subtarget) {
   MVT XLenVT = Subtarget.getXLenVT();
   RISCVMatInt::InstSeq Seq =
       RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
 
-  // If Imm is expensive to build, then we put it into constant pool.
-  if (Subtarget.useConstantPoolForLargeInts() &&
-      Seq.size() > Subtarget.getMaxBuildIntsCost())
-    return selectImmWithConstantPool(CurDAG, DL, VT, Imm, Subtarget);
-
   SDNode *Result = nullptr;
   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
   for (RISCVMatInt::Inst &Inst : Seq) {
     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
     switch (Inst.getOpndKind()) {
     case RISCVMatInt::Imm:
       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SDImm);
       break;

llvm/lib/Target/RISCV/RISCVISelLowering.h

 //===-- RISCVISelLowering.h - RISCV DAG Lowering Interface ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       LLVMContext &Context) const override;
   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
                       SelectionDAG &DAG) const override;
   SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
-  template <class NodeTy>
-  SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const;
 
   bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                          Type *Ty) const override {
     return true;
   }
   bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
 
private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
                         const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
                         RISCVCCAssignFn Fn) const;
   void analyzeOutputArgs(MachineFunction &MF, CCState &CCInfo,
                          const SmallVectorImpl<ISD::OutputArg> &Outs,
                          bool IsRet, CallLoweringInfo *CLI,
                          RISCVCCAssignFn Fn) const;
 
+  template <class NodeTy>
+  SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const;
   SDValue getStaticTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG,
                            bool UseGOT) const;
   SDValue getDynamicTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG) const;
 
   SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerJumpTable(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   }
 
   setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
                       ISD::JumpTable},
                      XLenVT, Custom);
 
   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
 
+  if (Subtarget.is64Bit())
+    setOperationAction(ISD::Constant, MVT::i64, Custom);
+
   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
   // Unfortunately this can't be determined just from the ISA naming string.
   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
                      Subtarget.is64Bit() ? Legal : Custom);
 
   setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
   if (Subtarget.is64Bit())
SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
   assert(NewVT.isValid() &&
          "Expecting equally-sized RVV vector types to be legal");
   StoredVal = DAG.getBitcast(NewVT, StoredVal);
   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
                       Store->getPointerInfo(), Store->getOriginalAlign(),
                       Store->getMemOperand()->getFlags());
 }
 
+static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
+                             const RISCVSubtarget &Subtarget) {
+  assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
+
+  // Only do this if some constants should use the constant pool.
+  if (!Subtarget.useConstantPoolForLargeInts())
+    return Op;
+
+  int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
+
+  // All simm32 constants should be handled by isel.
+  if (isInt<32>(Imm))
+    return Op;
+
+  RISCVMatInt::InstSeq Seq =
+      RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
+  if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
+    return Op;
+
+  // Expand to a constant pool using the default expansion code.
+  return SDValue();
+}
+
 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                             SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default:
     report_fatal_error("unimplemented operand");
   case ISD::GlobalAddress:
     return lowerGlobalAddress(Op, DAG);
   case ISD::BlockAddress:
     return lowerBlockAddress(Op, DAG);
   case ISD::ConstantPool:
     return lowerConstantPool(Op, DAG);
   case ISD::JumpTable:
     return lowerJumpTable(Op, DAG);
   case ISD::GlobalTLSAddress:
     return lowerGlobalTLSAddress(Op, DAG);
+  case ISD::Constant:
+    return lowerConstant(Op, DAG, Subtarget);
   case ISD::SELECT:
     return lowerSELECT(Op, DAG);
   case ISD::BRCOND:
     return lowerBRCOND(Op, DAG);
   case ISD::VASTART:
     return lowerVASTART(Op, DAG);
   case ISD::FRAMEADDR:
     return lowerFRAMEADDR(Op, DAG);
case CodeModel::Medium: {
     // the address space. This generates the pattern (PseudoLLA sym), which
     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
   }
   }
 }
 
 template SDValue RISCVTargetLowering::getAddr<GlobalAddressSDNode>(
     GlobalAddressSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
 template SDValue RISCVTargetLowering::getAddr<BlockAddressSDNode>(
     BlockAddressSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
 template SDValue RISCVTargetLowering::getAddr<ConstantPoolSDNode>(
     ConstantPoolSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
 template SDValue RISCVTargetLowering::getAddr<JumpTableSDNode>(
     JumpTableSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
 

llvm/test/CodeGen/RISCV/rv64zbp-intrinsic.ll

 }
 
 ; The second OR is redundant with the first. Make sure we remove it.
 define i64 @gorci64_knownbits(i64 %a) nounwind {
 ; RV64ZBP-LABEL: gorci64_knownbits:
 ; RV64ZBP:       # %bb.0:
 ; RV64ZBP-NEXT:    lui a1, %hi(.LCPI54_0)
 ; RV64ZBP-NEXT:    ld a1, %lo(.LCPI54_0)(a1)
+; RV64ZBP-NEXT:    lui a2, %hi(.LCPI54_1)
+; RV64ZBP-NEXT:    ld a2, %lo(.LCPI54_1)(a2)
 ; RV64ZBP-NEXT:    or a0, a0, a1
 ; RV64ZBP-NEXT:    orc32 a0, a0
+; RV64ZBP-NEXT:    or a0, a0, a2
 ; RV64ZBP-NEXT:    ret
   %tmp = or i64 %a, 72624976668147840 ; 0x102040810204080
   %tmp2 = call i64 @llvm.riscv.gorc.i64(i64 %tmp, i64 32)
   %tmp3 = or i64 %tmp2, 1234624599046636680 ; 0x1122448811224488
   ret i64 %tmp3
 }
 
 define i64 @orchi64(i64 %a) nounwind {

llvm/test/CodeGen/RISCV/rv64zbp.ll

 ; RV64I-NEXT:    srli a1, a3, 2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBP-LABEL: gorc2b_i64:
 ; RV64ZBP:       # %bb.0:
-; RV64ZBP-NEXT:    srli a1, a0, 2
-; RV64ZBP-NEXT:    or a1, a1, a0
-; RV64ZBP-NEXT:    orc2.n a0, a0
-; RV64ZBP-NEXT:    lui a2, %hi(.LCPI26_0)
-; RV64ZBP-NEXT:    ld a2, %lo(.LCPI26_0)(a2)
+; RV64ZBP-NEXT:    lui a1, %hi(.LCPI26_0)
+; RV64ZBP-NEXT:    ld a1, %lo(.LCPI26_0)(a1)
+; RV64ZBP-NEXT:    srli a2, a0, 2
+; RV64ZBP-NEXT:    and a2, a2, a1
 ; RV64ZBP-NEXT:    lui a3, %hi(.LCPI26_1)
 ; RV64ZBP-NEXT:    ld a3, %lo(.LCPI26_1)(a3)
-; RV64ZBP-NEXT:    slli a1, a1, 2
-; RV64ZBP-NEXT:    and a1, a1, a2
-; RV64ZBP-NEXT:    srli a2, a0, 2
+; RV64ZBP-NEXT:    or a2, a2, a0
+; RV64ZBP-NEXT:    orc2.n a0, a0
+; RV64ZBP-NEXT:    slli a2, a2, 2
 ; RV64ZBP-NEXT:    and a2, a2, a3
-; RV64ZBP-NEXT:    or a0, a2, a0
-; RV64ZBP-NEXT:    or a0, a0, a1
+; RV64ZBP-NEXT:    srli a3, a0, 2
+; RV64ZBP-NEXT:    and a1, a3, a1
+; RV64ZBP-NEXT:    or a0, a1, a0
+; RV64ZBP-NEXT:    or a0, a0, a2
 ; RV64ZBP-NEXT:    ret
   %and1 = shl i64 %a, 2
   %shl1 = and i64 %and1, -3689348814741910324
   %and1b = lshr i64 %a, 2
   %shr1 = and i64 %and1b, 3689348814741910323
   %or1 = or i64 %shr1, %a
   %or1b = or i64 %or1, %shl1
   %and2 = shl i64 %or1b, 2

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll

 ; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
 ; RV32-NEXT:    vle8.v v8, (a1)
 ; RV32-NEXT:    vse8.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: buildvec_seq_v16i8_v2i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a1, %hi(.LCPI24_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI24_0)(a1)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI24_0)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    vmv.v.x v8, a1
+; RV64-NEXT:    vlse64.v v8, (a1), zero
 ; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
 ; RV64-NEXT:    vse8.v v8, (a0)
 ; RV64-NEXT:    ret
   store <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8>, <16 x i8>* %x
   ret void
 }
 
 define void @buildvec_seq2_v16i8_v2i64(<16 x i8>* %x) {

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll

 ; RV32-NEXT:    vle32.v v9, (a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
 ; RV32-NEXT:    vsrl.vv v8, v8, v9
 ; RV32-NEXT:    vse64.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mulhu_v2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, %hi(.LCPI55_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI55_0)(a1)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    lui a2, %hi(.LCPI55_1)
-; RV64-NEXT:    ld a2, %lo(.LCPI55_1)(a2)
-; RV64-NEXT:    vmv.v.x v8, a1
+; RV64-NEXT:    lui a1, %hi(.LCPI55_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI55_0)
+; RV64-NEXT:    vlse64.v v8, (a1), zero
+; RV64-NEXT:    lui a1, %hi(.LCPI55_1)
+; RV64-NEXT:    ld a1, %lo(.LCPI55_1)(a1)
 ; RV64-NEXT:    vle64.v v9, (a0)
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    vmv.s.x v8, a1
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; RV64-NEXT:    vmulhu.vv v8, v9, v8
 ; RV64-NEXT:    vid.v v9
 ; RV64-NEXT:    vadd.vi v9, v9, 1
 ; RV64-NEXT:    vsrl.vv v8, v8, v9
 ; RV64-NEXT:    vse64.v v8, (a0)
 ; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x

 ; RV32-NEXT:    vsrl.vi v9, v8, 31
 ; RV32-NEXT:    vsra.vi v8, v8, 1
 ; RV32-NEXT:    vadd.vv v8, v8, v9
 ; RV32-NEXT:    vse32.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mulhs_v4i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, %hi(.LCPI58_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI58_0)(a1)
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
 ; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    lui a1, %hi(.LCPI58_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI58_0)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    vmv.v.x v9, a1
+; RV64-NEXT:    vlse64.v v9, (a1), zero
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
 ; RV64-NEXT:    vmulh.vv v8, v8, v9
 ; RV64-NEXT:    vsra.vi v8, v8, 1
 ; RV64-NEXT:    vsrl.vi v9, v8, 31
 ; RV64-NEXT:    vadd.vv v8, v8, v9
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x

 ; RV32-NEXT:    li a1, 63
 ; RV32-NEXT:    vsrl.vx v9, v10, a1
 ; RV32-NEXT:    vadd.vv v8, v8, v9
 ; RV32-NEXT:    vse64.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mulhs_v2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, %hi(.LCPI59_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI59_0)(a1)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    lui a2, %hi(.LCPI59_1)
-; RV64-NEXT:    ld a2, %lo(.LCPI59_1)(a2)
-; RV64-NEXT:    vmv.v.x v8, a1
+; RV64-NEXT:    lui a1, %hi(.LCPI59_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI59_0)
+; RV64-NEXT:    vlse64.v v8, (a1), zero
+; RV64-NEXT:    lui a1, %hi(.LCPI59_1)
+; RV64-NEXT:    ld a1, %lo(.LCPI59_1)(a1)
 ; RV64-NEXT:    vle64.v v9, (a0)
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    vmv.s.x v8, a1
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; RV64-NEXT:    vmulh.vv v8, v9, v8
 ; RV64-NEXT:    vid.v v10
 ; RV64-NEXT:    vrsub.vi v11, v10, 0
 ; RV64-NEXT:    vmadd.vv v11, v9, v8
 ; RV64-NEXT:    li a1, 63
 ; RV64-NEXT:    vsrl.vx v8, v11, a1
 ; RV64-NEXT:    vsra.vv v9, v11, v10

 ; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v9, (a1)
 ; LMULMAX1-RV64-NEXT:    vmv.v.i v10, 0
 ; LMULMAX1-RV64-NEXT:    li a2, -1
 ; LMULMAX1-RV64-NEXT:    slli a2, a2, 63
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v10, a2
 ; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_0)
-; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_0)(a2)
-; LMULMAX1-RV64-NEXT:    lui a3, %hi(.LCPI132_1)
-; LMULMAX1-RV64-NEXT:    ld a3, %lo(.LCPI132_1)(a3)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_0)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
-; LMULMAX1-RV64-NEXT:    vmv.v.x v11, a2
+; LMULMAX1-RV64-NEXT:    vlse64.v v11, (a2), zero
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_1)
+; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_1)(a2)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
-; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a3
+; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a2
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v11, v9, v11
 ; LMULMAX1-RV64-NEXT:    vsub.vv v9, v9, v11
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_2)
-; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_2)(a2)
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v11
 ; LMULMAX1-RV64-NEXT:    vid.v v10
-; LMULMAX1-RV64-NEXT:    vmv.v.x v11, a2
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_2)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_2)
+; LMULMAX1-RV64-NEXT:    vlse64.v v11, (a2), zero
 ; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_3)
 ; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_3)(a2)
 ; LMULMAX1-RV64-NEXT:    vadd.vi v12, v10, 2
 ; LMULMAX1-RV64-NEXT:    vsrl.vv v9, v9, v12
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a2
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v8, v8, v11

 ; LMULMAX2-RV32-NEXT:    vsrl.vi v10, v8, 31
 ; LMULMAX2-RV32-NEXT:    vsra.vi v8, v8, 1
 ; LMULMAX2-RV32-NEXT:    vadd.vv v8, v8, v10
 ; LMULMAX2-RV32-NEXT:    vse32.v v8, (a0)
 ; LMULMAX2-RV32-NEXT:    ret
 ;
 ; LMULMAX2-RV64-LABEL: mulhs_v8i32:
 ; LMULMAX2-RV64:       # %bb.0:
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI135_0)
-; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI135_0)(a1)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
 ; LMULMAX2-RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI135_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI135_0)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; LMULMAX2-RV64-NEXT:    vmv.v.x v10, a1
+; LMULMAX2-RV64-NEXT:    vlse64.v v10, (a1), zero
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
 ; LMULMAX2-RV64-NEXT:    vmulh.vv v8, v8, v10
 ; LMULMAX2-RV64-NEXT:    vsra.vi v8, v8, 1
 ; LMULMAX2-RV64-NEXT:    vsrl.vi v10, v8, 31
 ; LMULMAX2-RV64-NEXT:    vadd.vv v8, v8, v10
 ; LMULMAX2-RV64-NEXT:    vse32.v v8, (a0)
 ; LMULMAX2-RV64-NEXT:    ret
 ;

 ;
 ; LMULMAX2-RV64-LABEL: mulhs_v4i64:
 ; LMULMAX2-RV64:       # %bb.0:
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
 ; LMULMAX2-RV64-NEXT:    vle64.v v8, (a0)
 ; LMULMAX2-RV64-NEXT:    li a1, 5
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 1, e8, mf8, ta, mu
 ; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
-; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI136_0)(a1)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; LMULMAX2-RV64-NEXT:    lui a2, %hi(.LCPI136_1)
-; LMULMAX2-RV64-NEXT:    ld a2, %lo(.LCPI136_1)(a2)
-; LMULMAX2-RV64-NEXT:    vmv.v.x v10, a1
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI136_0)
+; LMULMAX2-RV64-NEXT:    vlse64.v v10, (a1), zero
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI136_1)
+; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI136_1)(a1)
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, -1
 ; LMULMAX2-RV64-NEXT:    vmerge.vim v12, v12, 0, v0
-; LMULMAX2-RV64-NEXT:    vmerge.vxm v10, v10, a2, v0
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v10, v10, a1, v0
 ; LMULMAX2-RV64-NEXT:    vmulh.vv v10, v8, v10
 ; LMULMAX2-RV64-NEXT:    vmacc.vv v10, v8, v12
 ; LMULMAX2-RV64-NEXT:    li a1, 63
 ; LMULMAX2-RV64-NEXT:    vsrl.vx v8, v10, a1
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, 1
 ; LMULMAX2-RV64-NEXT:    vmerge.vim v12, v12, 0, v0
 ; LMULMAX2-RV64-NEXT:    vsra.vv v10, v10, v12
 ; LMULMAX2-RV64-NEXT:    vadd.vv v8, v10, v8

 ; LMULMAX1-RV32-NEXT:    vdiv.vv v9, v9, v10
 ; LMULMAX1-RV32-NEXT:    vdiv.vv v8, v8, v10
 ; LMULMAX1-RV32-NEXT:    vse64.v v8, (a0)
 ; LMULMAX1-RV32-NEXT:    vse64.v v9, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: mulhs_v4i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
-; LMULMAX1-RV64-NEXT:    ld a1, %lo(.LCPI136_0)(a1)
 ; LMULMAX1-RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vle64.v v8, (a0)
-; LMULMAX1-RV64-NEXT:    vmv.v.x v9, a1
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
+; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI136_0)
+; LMULMAX1-RV64-NEXT:    vlse64.v v9, (a1), zero
 ; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI136_1)
 ; LMULMAX1-RV64-NEXT:    ld a1, %lo(.LCPI136_1)(a1)
 ; LMULMAX1-RV64-NEXT:    addi a2, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v10, (a2)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v9, a1
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vmulh.vv v11, v10, v9

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll

 ; RV32-LMULMAX4-NEXT:    vmv.s.x v0, a0
 ; RV32-LMULMAX4-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
 ; RV32-LMULMAX4-NEXT:    vslideup.vi v0, v8, 1
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_v64i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64-LMULMAX4-NEXT:    ld a0, %lo(.LCPI19_0)(a0)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI19_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_v64i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, 748388
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
 ; RV32-LMULMAX8-NEXT:    vsetivli zero, 2, e32, mf2, ta, mu
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v8, a0
 ; RV32-LMULMAX8-NEXT:    lui a0, 748384
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v0, a0
 ; RV32-LMULMAX8-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
 ; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v8, 1
 ; RV32-LMULMAX8-NEXT:    ret
 ;
 ; RV64-LMULMAX8-LABEL: buildvec_mask_v64i1:
 ; RV64-LMULMAX8:       # %bb.0:
 ; RV64-LMULMAX8-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64-LMULMAX8-NEXT:    ld a0, %lo(.LCPI19_0)(a0)
+; RV64-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI19_0)
 ; RV64-LMULMAX8-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX8-NEXT:    vlse64.v v0, (a0), zero
 ; RV64-LMULMAX8-NEXT:    ret
   ret <64 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
 }
 
 define <128 x i1> @buildvec_mask_v128i1() {
 ; RV32-LMULMAX1-LABEL: buildvec_mask_v128i1:
 ; RV32-LMULMAX1:       # %bb.0:
 ; RV32-LMULMAX1-NEXT:    li a0, 1776

 ; RV32-LMULMAX4-NEXT:    vmv.s.x v8, a0
 ; RV32-LMULMAX4-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
 ; RV32-LMULMAX4-NEXT:    vslideup.vi v8, v9, 1
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_v128i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI20_0)
-; RV64-LMULMAX4-NEXT:    ld a0, %lo(.LCPI20_0)(a0)
-; RV64-LMULMAX4-NEXT:    lui a1, %hi(.LCPI20_1)
-; RV64-LMULMAX4-NEXT:    ld a1, %lo(.LCPI20_1)(a1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI20_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
-; RV64-LMULMAX4-NEXT:    vmv.s.x v8, a1
+; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI20_1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI20_1)
+; RV64-LMULMAX4-NEXT:    vlse64.v v8, (a0), zero
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_v128i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, 748388
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
 ; RV32-LMULMAX8-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v8, a0

 ; RV32-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_1)
 ; RV32-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_1)
 ; RV32-LMULMAX4-NEXT:    vlm.v v8, (a0)
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_optsize_v128i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_0)
-; RV64-LMULMAX4-NEXT:    ld a0, %lo(.LCPI21_0)(a0)
-; RV64-LMULMAX4-NEXT:    lui a1, %hi(.LCPI21_1)
-; RV64-LMULMAX4-NEXT:    ld a1, %lo(.LCPI21_1)(a1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
-; RV64-LMULMAX4-NEXT:    vmv.s.x v8, a1
+; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_1)
+; RV64-LMULMAX4-NEXT:    vlse64.v v8, (a0), zero
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_optsize_v128i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, %hi(.LCPI21_0)
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI21_0)
 ; RV32-LMULMAX8-NEXT:    li a1, 128
 ; RV32-LMULMAX8-NEXT:    vsetvli zero, a1, e8, m8, ta, mu