[RISCV] Move creation of constant pools from isel to lowering.

[craig.topper]

Summary

This simplifies the isel code by removing the manual load creation.
It also improves our ability to use 0 strided loads for vector splats.

There is an assumption here that Mask and ShiftedMask constants are
cheap enough that they don't become constant pool loads so that our
isel optimizations involving And still work. I believe those constants
are 3 instructions in the worst case.

The rv64zbp-intrinsic.ll changes is a regression caused by intrinsics
being expanded to RISCVISD also occuring during lowering. So the optimizations
were only happening during the last DAGCombine, which can't see through the
load. I believe we can fix this test by implementing
TargetLowering::getTargetConstantFromLoad for RISC-V or by adding the intrinsic
to computeKnownBitsForTargetNode to enable earlier DAG combine. Since Zbp is not
a ratified extension, I don't view these as blocking this patch.

[craig.topper]

Move getAddr back to being a private member of RISCVTargetLowering

[wangpc]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

3644–3645

I think these instantiations can be removed now.

[craig.topper]

Remove unneeded temporary instantiations

[reames]

LGTM

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

2973

Minor: This check should be redundant except maybe as a minor compile time win assuming the count threshold just below is at least 2.

[craig.topper]

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

2973

Agreed, I mainly did it for the compile time win. getMaxBuildIntsCost has a max with 2 in it.

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

case RISCV::FSD:
     BaseOpIdx = 1;
     OffsetOpIdx = 2;
     return true;
   }
 
   return false;
 }
 
-static SDNode *selectImmWithConstantPool(SelectionDAG *CurDAG, const SDLoc &DL,
-                                         const MVT VT, int64_t Imm,
-                                         const RISCVSubtarget &Subtarget) {
-  assert(VT == MVT::i64 && "Expecting MVT::i64");
-  const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
-  ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(CurDAG->getConstantPool(
-      ConstantInt::get(EVT(VT).getTypeForEVT(*CurDAG->getContext()), Imm), VT));
-  SDValue Addr = TLI->getAddr(CP, *CurDAG);
-  SDValue Offset = CurDAG->getTargetConstant(0, DL, VT);
-  // Since there is no data race, the chain can be the entry node.
-  SDNode *Load = CurDAG->getMachineNode(RISCV::LD, DL, VT, Addr, Offset,
-                                        CurDAG->getEntryNode());
-  MachineFunction &MF = CurDAG->getMachineFunction();
-  MachineMemOperand *MemOp = MF.getMachineMemOperand(
-      MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
-      LLT(VT), CP->getAlign());
-  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Load), {MemOp});
-  return Load;
-}
-
 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
                          int64_t Imm, const RISCVSubtarget &Subtarget) {
   MVT XLenVT = Subtarget.getXLenVT();
   RISCVMatInt::InstSeq Seq =
       RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
 
-  // If Imm is expensive to build, then we put it into constant pool.
-  if (Subtarget.useConstantPoolForLargeInts() &&
-      Seq.size() > Subtarget.getMaxBuildIntsCost())
-    return selectImmWithConstantPool(CurDAG, DL, VT, Imm, Subtarget);
-
   SDNode *Result = nullptr;
   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
   for (RISCVMatInt::Inst &Inst : Seq) {
     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
     switch (Inst.getOpndKind()) {
     case RISCVMatInt::Imm:
       Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SDImm);
       break;

llvm/lib/Target/RISCV/RISCVISelLowering.h

bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       LLVMContext &Context) const override;
   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
                       const SmallVectorImpl<ISD::OutputArg> &Outs,
                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
                       SelectionDAG &DAG) const override;
   SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
-  template <class NodeTy>
-  SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const;
 
   bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                          Type *Ty) const override {
     return true;
   }
   bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
 
private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
                         const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
                         RISCVCCAssignFn Fn) const;
   void analyzeOutputArgs(MachineFunction &MF, CCState &CCInfo,
                          const SmallVectorImpl<ISD::OutputArg> &Outs,
                          bool IsRet, CallLoweringInfo *CLI,
                          RISCVCCAssignFn Fn) const;
 
+  template <class NodeTy>
+  SDValue getAddr(NodeTy *N, SelectionDAG &DAG, bool IsLocal = true) const;
   SDValue getStaticTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG,
                            bool UseGOT) const;
   SDValue getDynamicTLSAddr(GlobalAddressSDNode *N, SelectionDAG &DAG) const;
 
   SDValue lowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerJumpTable(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   }
 
   setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
                       ISD::JumpTable},
                      XLenVT, Custom);
 
   setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
 
+  if (Subtarget.is64Bit())
+    setOperationAction(ISD::Constant, MVT::i64, Custom);
+
   // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
   // Unfortunately this can't be determined just from the ISA naming string.
   setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
                      Subtarget.is64Bit() ? Legal : Custom);
 
   setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
   if (Subtarget.is64Bit())
SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
   assert(NewVT.isValid() &&
          "Expecting equally-sized RVV vector types to be legal");
   StoredVal = DAG.getBitcast(NewVT, StoredVal);
   return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
                       Store->getPointerInfo(), Store->getOriginalAlign(),
                       Store->getMemOperand()->getFlags());
 }
 
+static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
+                             const RISCVSubtarget &Subtarget) {
+  assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
+
+  int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
+
+  // All simm32 constants should be handled by isel.
+  // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
+  // this check redundant, but small immediates are common so this check
+  // should have better compile time.
+  if (isInt<32>(Imm))
+    return Op;
+
+  // We only need to cost the immediate, if constant pool lowering is enabled.
+  if (!Subtarget.useConstantPoolForLargeInts())
+    return Op;
+
+  RISCVMatInt::InstSeq Seq =
+      RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
+  if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
+    return Op;
+
+  // Expand to a constant pool using the default expansion code.
+  return SDValue();
+}
+
 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                             SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default:
     report_fatal_error("unimplemented operand");
   case ISD::GlobalAddress:
     return lowerGlobalAddress(Op, DAG);
   case ISD::BlockAddress:
     return lowerBlockAddress(Op, DAG);
   case ISD::ConstantPool:
     return lowerConstantPool(Op, DAG);
   case ISD::JumpTable:
     return lowerJumpTable(Op, DAG);
   case ISD::GlobalTLSAddress:
     return lowerGlobalTLSAddress(Op, DAG);
+  case ISD::Constant:
+    return lowerConstant(Op, DAG, Subtarget);
   case ISD::SELECT:
     return lowerSELECT(Op, DAG);
   case ISD::BRCOND:
     return lowerBRCOND(Op, DAG);
   case ISD::VASTART:
     return lowerVASTART(Op, DAG);
   case ISD::FRAMEADDR:
     return lowerFRAMEADDR(Op, DAG);
case CodeModel::Medium: {
     // Generate a sequence for accessing addresses within any 2GiB range within
     // the address space. This generates the pattern (PseudoLLA sym), which
     // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
     SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
     return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
   }
   }
 }
 
-template SDValue RISCVTargetLowering::getAddr<GlobalAddressSDNode>(
-    GlobalAddressSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
-template SDValue RISCVTargetLowering::getAddr<BlockAddressSDNode>(
-    BlockAddressSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
-template SDValue RISCVTargetLowering::getAddr<ConstantPoolSDNode>(
-    ConstantPoolSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
-template SDValue RISCVTargetLowering::getAddr<JumpTableSDNode>(
-    JumpTableSDNode *N, SelectionDAG &DAG, bool IsLocal) const;
-
 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
   SDLoc DL(Op);
   GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
   assert(N->getOffset() == 0 && "unexpected offset in global node");
 
   const GlobalValue *GV = N->getGlobal();
   bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
   return getAddr(N, DAG, IsLocal);

llvm/test/CodeGen/RISCV/rv64zbp-intrinsic.ll

 }
 
 ; The second OR is redundant with the first. Make sure we remove it.
 define i64 @gorci64_knownbits(i64 %a) nounwind {
 ; RV64ZBP-LABEL: gorci64_knownbits:
 ; RV64ZBP:       # %bb.0:
 ; RV64ZBP-NEXT:    lui a1, %hi(.LCPI54_0)
 ; RV64ZBP-NEXT:    ld a1, %lo(.LCPI54_0)(a1)
+; RV64ZBP-NEXT:    lui a2, %hi(.LCPI54_1)
+; RV64ZBP-NEXT:    ld a2, %lo(.LCPI54_1)(a2)
 ; RV64ZBP-NEXT:    or a0, a0, a1
 ; RV64ZBP-NEXT:    orc32 a0, a0
+; RV64ZBP-NEXT:    or a0, a0, a2
 ; RV64ZBP-NEXT:    ret
   %tmp = or i64 %a, 72624976668147840 ; 0x102040810204080
   %tmp2 = call i64 @llvm.riscv.gorc.i64(i64 %tmp, i64 32)
   %tmp3 = or i64 %tmp2, 1234624599046636680 ; 0x1122448811224488
   ret i64 %tmp3
 }
 
 define i64 @orchi64(i64 %a) nounwind {

llvm/test/CodeGen/RISCV/rv64zbp.ll

 ; RV64I-NEXT:    srli a1, a3, 2
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
 ; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
 ; RV64ZBP-LABEL: gorc2b_i64:
 ; RV64ZBP:       # %bb.0:
-; RV64ZBP-NEXT:    srli a1, a0, 2
-; RV64ZBP-NEXT:    or a1, a1, a0
-; RV64ZBP-NEXT:    orc2.n a0, a0
-; RV64ZBP-NEXT:    lui a2, %hi(.LCPI26_0)
-; RV64ZBP-NEXT:    ld a2, %lo(.LCPI26_0)(a2)
+; RV64ZBP-NEXT:    lui a1, %hi(.LCPI26_0)
+; RV64ZBP-NEXT:    ld a1, %lo(.LCPI26_0)(a1)
+; RV64ZBP-NEXT:    srli a2, a0, 2
+; RV64ZBP-NEXT:    and a2, a2, a1
 ; RV64ZBP-NEXT:    lui a3, %hi(.LCPI26_1)
 ; RV64ZBP-NEXT:    ld a3, %lo(.LCPI26_1)(a3)
-; RV64ZBP-NEXT:    slli a1, a1, 2
-; RV64ZBP-NEXT:    and a1, a1, a2
-; RV64ZBP-NEXT:    srli a2, a0, 2
+; RV64ZBP-NEXT:    or a2, a2, a0
+; RV64ZBP-NEXT:    orc2.n a0, a0
+; RV64ZBP-NEXT:    slli a2, a2, 2
 ; RV64ZBP-NEXT:    and a2, a2, a3
-; RV64ZBP-NEXT:    or a0, a2, a0
-; RV64ZBP-NEXT:    or a0, a0, a1
+; RV64ZBP-NEXT:    srli a3, a0, 2
+; RV64ZBP-NEXT:    and a1, a3, a1
+; RV64ZBP-NEXT:    or a0, a1, a0
+; RV64ZBP-NEXT:    or a0, a0, a2
 ; RV64ZBP-NEXT:    ret
   %and1 = shl i64 %a, 2
   %shl1 = and i64 %and1, -3689348814741910324
   %and1b = lshr i64 %a, 2
   %shr1 = and i64 %and1b, 3689348814741910323
   %or1 = or i64 %shr1, %a
   %or1b = or i64 %or1, %shl1
   %and2 = shl i64 %or1b, 2

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll

 ; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
 ; RV32-NEXT:    vle8.v v8, (a1)
 ; RV32-NEXT:    vse8.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: buildvec_seq_v16i8_v2i64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    lui a1, %hi(.LCPI24_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI24_0)(a1)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI24_0)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    vmv.v.x v8, a1
+; RV64-NEXT:    vlse64.v v8, (a1), zero
 ; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
 ; RV64-NEXT:    vse8.v v8, (a0)
 ; RV64-NEXT:    ret
   store <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8>, <16 x i8>* %x
   ret void
 }
 
 define void @buildvec_seq2_v16i8_v2i64(<16 x i8>* %x) {

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int.ll

 ; RV32-NEXT:    vle32.v v9, (a1)
 ; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
 ; RV32-NEXT:    vsrl.vv v8, v8, v9
 ; RV32-NEXT:    vse64.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mulhu_v2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, %hi(.LCPI55_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI55_0)(a1)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    lui a2, %hi(.LCPI55_1)
-; RV64-NEXT:    ld a2, %lo(.LCPI55_1)(a2)
-; RV64-NEXT:    vmv.v.x v8, a1
+; RV64-NEXT:    lui a1, %hi(.LCPI55_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI55_0)
+; RV64-NEXT:    vlse64.v v8, (a1), zero
+; RV64-NEXT:    lui a1, %hi(.LCPI55_1)
+; RV64-NEXT:    ld a1, %lo(.LCPI55_1)(a1)
 ; RV64-NEXT:    vle64.v v9, (a0)
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    vmv.s.x v8, a1
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; RV64-NEXT:    vmulhu.vv v8, v9, v8
 ; RV64-NEXT:    vid.v v9
 ; RV64-NEXT:    vadd.vi v9, v9, 1
 ; RV64-NEXT:    vsrl.vv v8, v8, v9
 ; RV64-NEXT:    vse64.v v8, (a0)
 ; RV64-NEXT:    ret
   %a = load <2 x i64>, <2 x i64>* %x

 ; RV32-NEXT:    vsrl.vi v9, v8, 31
 ; RV32-NEXT:    vsra.vi v8, v8, 1
 ; RV32-NEXT:    vadd.vv v8, v8, v9
 ; RV32-NEXT:    vse32.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mulhs_v4i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, %hi(.LCPI58_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI58_0)(a1)
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
 ; RV64-NEXT:    vle32.v v8, (a0)
+; RV64-NEXT:    lui a1, %hi(.LCPI58_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI58_0)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    vmv.v.x v9, a1
+; RV64-NEXT:    vlse64.v v9, (a1), zero
 ; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
 ; RV64-NEXT:    vmulh.vv v8, v8, v9
 ; RV64-NEXT:    vsra.vi v8, v8, 1
 ; RV64-NEXT:    vsrl.vi v9, v8, 31
 ; RV64-NEXT:    vadd.vv v8, v8, v9
 ; RV64-NEXT:    vse32.v v8, (a0)
 ; RV64-NEXT:    ret
   %a = load <4 x i32>, <4 x i32>* %x

 ; RV32-NEXT:    li a1, 63
 ; RV32-NEXT:    vsrl.vx v9, v10, a1
 ; RV32-NEXT:    vadd.vv v8, v8, v9
 ; RV32-NEXT:    vse64.v v8, (a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: mulhs_v2i64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lui a1, %hi(.LCPI59_0)
-; RV64-NEXT:    ld a1, %lo(.LCPI59_0)(a1)
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
-; RV64-NEXT:    lui a2, %hi(.LCPI59_1)
-; RV64-NEXT:    ld a2, %lo(.LCPI59_1)(a2)
-; RV64-NEXT:    vmv.v.x v8, a1
+; RV64-NEXT:    lui a1, %hi(.LCPI59_0)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI59_0)
+; RV64-NEXT:    vlse64.v v8, (a1), zero
+; RV64-NEXT:    lui a1, %hi(.LCPI59_1)
+; RV64-NEXT:    ld a1, %lo(.LCPI59_1)(a1)
 ; RV64-NEXT:    vle64.v v9, (a0)
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
-; RV64-NEXT:    vmv.s.x v8, a2
+; RV64-NEXT:    vmv.s.x v8, a1
 ; RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; RV64-NEXT:    vmulh.vv v8, v9, v8
 ; RV64-NEXT:    vid.v v10
 ; RV64-NEXT:    vrsub.vi v11, v10, 0
 ; RV64-NEXT:    vmadd.vv v11, v9, v8
 ; RV64-NEXT:    li a1, 63
 ; RV64-NEXT:    vsrl.vx v8, v11, a1
 ; RV64-NEXT:    vsra.vv v9, v11, v10

 ; LMULMAX1-RV64-NEXT:    addi a1, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v9, (a1)
 ; LMULMAX1-RV64-NEXT:    vmv.v.i v10, 0
 ; LMULMAX1-RV64-NEXT:    li a2, -1
 ; LMULMAX1-RV64-NEXT:    slli a2, a2, 63
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v10, a2
 ; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_0)
-; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_0)(a2)
-; LMULMAX1-RV64-NEXT:    lui a3, %hi(.LCPI132_1)
-; LMULMAX1-RV64-NEXT:    ld a3, %lo(.LCPI132_1)(a3)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_0)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
-; LMULMAX1-RV64-NEXT:    vmv.v.x v11, a2
+; LMULMAX1-RV64-NEXT:    vlse64.v v11, (a2), zero
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_1)
+; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_1)(a2)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
-; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a3
+; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a2
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v11, v9, v11
 ; LMULMAX1-RV64-NEXT:    vsub.vv v9, v9, v11
-; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_2)
-; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_2)(a2)
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v9, v9, v10
 ; LMULMAX1-RV64-NEXT:    vadd.vv v9, v9, v11
 ; LMULMAX1-RV64-NEXT:    vid.v v10
-; LMULMAX1-RV64-NEXT:    vmv.v.x v11, a2
+; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_2)
+; LMULMAX1-RV64-NEXT:    addi a2, a2, %lo(.LCPI132_2)
+; LMULMAX1-RV64-NEXT:    vlse64.v v11, (a2), zero
 ; LMULMAX1-RV64-NEXT:    lui a2, %hi(.LCPI132_3)
 ; LMULMAX1-RV64-NEXT:    ld a2, %lo(.LCPI132_3)(a2)
 ; LMULMAX1-RV64-NEXT:    vadd.vi v12, v10, 2
 ; LMULMAX1-RV64-NEXT:    vsrl.vv v9, v9, v12
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v11, a2
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vmulhu.vv v8, v8, v11

 ; LMULMAX2-RV32-NEXT:    vsrl.vi v10, v8, 31
 ; LMULMAX2-RV32-NEXT:    vsra.vi v8, v8, 1
 ; LMULMAX2-RV32-NEXT:    vadd.vv v8, v8, v10
 ; LMULMAX2-RV32-NEXT:    vse32.v v8, (a0)
 ; LMULMAX2-RV32-NEXT:    ret
 ;
 ; LMULMAX2-RV64-LABEL: mulhs_v8i32:
 ; LMULMAX2-RV64:       # %bb.0:
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI135_0)
-; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI135_0)(a1)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
 ; LMULMAX2-RV64-NEXT:    vle32.v v8, (a0)
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI135_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI135_0)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; LMULMAX2-RV64-NEXT:    vmv.v.x v10, a1
+; LMULMAX2-RV64-NEXT:    vlse64.v v10, (a1), zero
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
 ; LMULMAX2-RV64-NEXT:    vmulh.vv v8, v8, v10
 ; LMULMAX2-RV64-NEXT:    vsra.vi v8, v8, 1
 ; LMULMAX2-RV64-NEXT:    vsrl.vi v10, v8, 31
 ; LMULMAX2-RV64-NEXT:    vadd.vv v8, v8, v10
 ; LMULMAX2-RV64-NEXT:    vse32.v v8, (a0)
 ; LMULMAX2-RV64-NEXT:    ret
 ;

 ;
 ; LMULMAX2-RV64-LABEL: mulhs_v4i64:
 ; LMULMAX2-RV64:       # %bb.0:
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
 ; LMULMAX2-RV64-NEXT:    vle64.v v8, (a0)
 ; LMULMAX2-RV64-NEXT:    li a1, 5
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 1, e8, mf8, ta, mu
 ; LMULMAX2-RV64-NEXT:    vmv.s.x v0, a1
-; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
-; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI136_0)(a1)
 ; LMULMAX2-RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, mu
-; LMULMAX2-RV64-NEXT:    lui a2, %hi(.LCPI136_1)
-; LMULMAX2-RV64-NEXT:    ld a2, %lo(.LCPI136_1)(a2)
-; LMULMAX2-RV64-NEXT:    vmv.v.x v10, a1
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
+; LMULMAX2-RV64-NEXT:    addi a1, a1, %lo(.LCPI136_0)
+; LMULMAX2-RV64-NEXT:    vlse64.v v10, (a1), zero
+; LMULMAX2-RV64-NEXT:    lui a1, %hi(.LCPI136_1)
+; LMULMAX2-RV64-NEXT:    ld a1, %lo(.LCPI136_1)(a1)
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, -1
 ; LMULMAX2-RV64-NEXT:    vmerge.vim v12, v12, 0, v0
-; LMULMAX2-RV64-NEXT:    vmerge.vxm v10, v10, a2, v0
+; LMULMAX2-RV64-NEXT:    vmerge.vxm v10, v10, a1, v0
 ; LMULMAX2-RV64-NEXT:    vmulh.vv v10, v8, v10
 ; LMULMAX2-RV64-NEXT:    vmacc.vv v10, v8, v12
 ; LMULMAX2-RV64-NEXT:    li a1, 63
 ; LMULMAX2-RV64-NEXT:    vsrl.vx v8, v10, a1
 ; LMULMAX2-RV64-NEXT:    vmv.v.i v12, 1
 ; LMULMAX2-RV64-NEXT:    vmerge.vim v12, v12, 0, v0
 ; LMULMAX2-RV64-NEXT:    vsra.vv v10, v10, v12
 ; LMULMAX2-RV64-NEXT:    vadd.vv v8, v10, v8

 ; LMULMAX1-RV32-NEXT:    vdiv.vv v9, v9, v10
 ; LMULMAX1-RV32-NEXT:    vdiv.vv v8, v8, v10
 ; LMULMAX1-RV32-NEXT:    vse64.v v8, (a0)
 ; LMULMAX1-RV32-NEXT:    vse64.v v9, (a1)
 ; LMULMAX1-RV32-NEXT:    ret
 ;
 ; LMULMAX1-RV64-LABEL: mulhs_v4i64:
 ; LMULMAX1-RV64:       # %bb.0:
-; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
-; LMULMAX1-RV64-NEXT:    ld a1, %lo(.LCPI136_0)(a1)
 ; LMULMAX1-RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vle64.v v8, (a0)
-; LMULMAX1-RV64-NEXT:    vmv.v.x v9, a1
+; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI136_0)
+; LMULMAX1-RV64-NEXT:    addi a1, a1, %lo(.LCPI136_0)
+; LMULMAX1-RV64-NEXT:    vlse64.v v9, (a1), zero
 ; LMULMAX1-RV64-NEXT:    lui a1, %hi(.LCPI136_1)
 ; LMULMAX1-RV64-NEXT:    ld a1, %lo(.LCPI136_1)(a1)
 ; LMULMAX1-RV64-NEXT:    addi a2, a0, 16
 ; LMULMAX1-RV64-NEXT:    vle64.v v10, (a2)
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, tu, mu
 ; LMULMAX1-RV64-NEXT:    vmv.s.x v9, a1
 ; LMULMAX1-RV64-NEXT:    vsetvli zero, zero, e64, m1, ta, mu
 ; LMULMAX1-RV64-NEXT:    vmulh.vv v11, v10, v9

llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll

 ; RV32-LMULMAX4-NEXT:    vmv.s.x v0, a0
 ; RV32-LMULMAX4-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
 ; RV32-LMULMAX4-NEXT:    vslideup.vi v0, v8, 1
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_v64i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64-LMULMAX4-NEXT:    ld a0, %lo(.LCPI19_0)(a0)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI19_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_v64i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, 748388
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
 ; RV32-LMULMAX8-NEXT:    vsetivli zero, 2, e32, mf2, ta, mu
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v8, a0
 ; RV32-LMULMAX8-NEXT:    lui a0, 748384
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, 1776
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v0, a0
 ; RV32-LMULMAX8-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
 ; RV32-LMULMAX8-NEXT:    vslideup.vi v0, v8, 1
 ; RV32-LMULMAX8-NEXT:    ret
 ;
 ; RV64-LMULMAX8-LABEL: buildvec_mask_v64i1:
 ; RV64-LMULMAX8:       # %bb.0:
 ; RV64-LMULMAX8-NEXT:    lui a0, %hi(.LCPI19_0)
-; RV64-LMULMAX8-NEXT:    ld a0, %lo(.LCPI19_0)(a0)
+; RV64-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI19_0)
 ; RV64-LMULMAX8-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX8-NEXT:    vmv.s.x v0, a0
+; RV64-LMULMAX8-NEXT:    vlse64.v v0, (a0), zero
 ; RV64-LMULMAX8-NEXT:    ret
   ret <64 x i1> <i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1>
 }
 
 define <128 x i1> @buildvec_mask_v128i1() {
 ; RV32-LMULMAX1-LABEL: buildvec_mask_v128i1:
 ; RV32-LMULMAX1:       # %bb.0:
 ; RV32-LMULMAX1-NEXT:    li a0, 1776

 ; RV32-LMULMAX4-NEXT:    vmv.s.x v8, a0
 ; RV32-LMULMAX4-NEXT:    vsetvli zero, zero, e32, mf2, tu, mu
 ; RV32-LMULMAX4-NEXT:    vslideup.vi v8, v9, 1
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_v128i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI20_0)
-; RV64-LMULMAX4-NEXT:    ld a0, %lo(.LCPI20_0)(a0)
-; RV64-LMULMAX4-NEXT:    lui a1, %hi(.LCPI20_1)
-; RV64-LMULMAX4-NEXT:    ld a1, %lo(.LCPI20_1)(a1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI20_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
-; RV64-LMULMAX4-NEXT:    vmv.s.x v8, a1
+; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI20_1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI20_1)
+; RV64-LMULMAX4-NEXT:    vlse64.v v8, (a0), zero
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_v128i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, 748388
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, -1793
 ; RV32-LMULMAX8-NEXT:    vsetivli zero, 4, e32, m1, ta, mu
 ; RV32-LMULMAX8-NEXT:    vmv.s.x v8, a0

 ; RV32-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_1)
 ; RV32-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_1)
 ; RV32-LMULMAX4-NEXT:    vlm.v v8, (a0)
 ; RV32-LMULMAX4-NEXT:    ret
 ;
 ; RV64-LMULMAX4-LABEL: buildvec_mask_optsize_v128i1:
 ; RV64-LMULMAX4:       # %bb.0:
 ; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_0)
-; RV64-LMULMAX4-NEXT:    ld a0, %lo(.LCPI21_0)(a0)
-; RV64-LMULMAX4-NEXT:    lui a1, %hi(.LCPI21_1)
-; RV64-LMULMAX4-NEXT:    ld a1, %lo(.LCPI21_1)(a1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_0)
 ; RV64-LMULMAX4-NEXT:    vsetivli zero, 1, e64, m1, ta, mu
-; RV64-LMULMAX4-NEXT:    vmv.s.x v0, a0
-; RV64-LMULMAX4-NEXT:    vmv.s.x v8, a1
+; RV64-LMULMAX4-NEXT:    vlse64.v v0, (a0), zero
+; RV64-LMULMAX4-NEXT:    lui a0, %hi(.LCPI21_1)
+; RV64-LMULMAX4-NEXT:    addi a0, a0, %lo(.LCPI21_1)
+; RV64-LMULMAX4-NEXT:    vlse64.v v8, (a0), zero
 ; RV64-LMULMAX4-NEXT:    ret
 ;
 ; RV32-LMULMAX8-LABEL: buildvec_mask_optsize_v128i1:
 ; RV32-LMULMAX8:       # %bb.0:
 ; RV32-LMULMAX8-NEXT:    lui a0, %hi(.LCPI21_0)
 ; RV32-LMULMAX8-NEXT:    addi a0, a0, %lo(.LCPI21_0)
 ; RV32-LMULMAX8-NEXT:    li a1, 128
 ; RV32-LMULMAX8-NEXT:    vsetvli zero, a1, e8, m8, ta, mu