Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -988,11 +988,14 @@ /// result and takes a list of operands. Opcode may be INTRINSIC_VOID, /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not /// less than FIRST_TARGET_MEMORY_OPCODE. - SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, - ArrayRef Ops, EVT MemVT, - MachinePointerInfo PtrInfo, unsigned Align = 0, - bool Vol = false, bool ReadMem = true, - bool WriteMem = true, unsigned Size = 0); + SDValue getMemIntrinsicNode( + unsigned Opcode, const SDLoc &dl, SDVTList VTList, + ArrayRef Ops, EVT MemVT, + MachinePointerInfo PtrInfo, + unsigned Align = 0, + MachineMemOperand::Flags Flags + = MachineMemOperand::MOLoad | MachineMemOperand::MOStore, + unsigned Size = 0); SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, EVT MemVT, Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -707,10 +707,8 @@ unsigned size = 0; // the size of the memory location // (taken from memVT if zero) unsigned align = 1; // alignment - bool vol = false; // is volatile? - bool readMem = false; // reads memory? - bool writeMem = false; // writes memory? + MachineMemOperand::Flags flags = MachineMemOperand::MONone; IntrinsicInfo() = default; }; Index: lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- lib/CodeGen/GlobalISel/IRTranslator.cpp +++ lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -852,13 +852,9 @@ TargetLowering::IntrinsicInfo Info; // TODO: Add a GlobalISel version of getTgtMemIntrinsic. if (TLI.getTgtMemIntrinsic(Info, CI, ID)) { - MachineMemOperand::Flags Flags = - Info.vol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; - Flags |= - Info.readMem ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore; uint64_t Size = Info.memVT.getStoreSize(); MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal), - Flags, Size, Info.align)); + Info.flags, Size, Info.align)); } return true; Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5776,26 +5776,20 @@ } SDValue SelectionDAG::getMemIntrinsicNode( - unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, - EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, - bool ReadMem, bool WriteMem, unsigned Size) { + unsigned Opcode, const SDLoc &SL, SDVTList VTList, ArrayRef Ops, + EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, + MachineMemOperand::Flags Flags, unsigned Size) { if (Align == 0) // Ensure that codegen never sees alignment 0 Align = getEVTAlignment(MemVT); - MachineFunction &MF = getMachineFunction(); - auto Flags = MachineMemOperand::MONone; - if (WriteMem) - Flags |= MachineMemOperand::MOStore; - if (ReadMem) - Flags |= MachineMemOperand::MOLoad; - if (Vol) - Flags |= MachineMemOperand::MOVolatile; if (!Size) Size = MemVT.getStoreSize(); + + MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, Flags, Size, Align); - return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); + return getMemIntrinsicNode(Opcode, SL, VTList, Ops, MemVT, MMO); } SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4239,11 +4239,10 @@ SDValue Result; if (IsTgtIntrinsic) { // This is target intrinsic that touches memory - Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), - VTs, Ops, Info.memVT, - MachinePointerInfo(Info.ptrVal, Info.offset), - Info.align, Info.vol, - Info.readMem, Info.writeMem, Info.size); + Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, + Ops, Info.memVT, + MachinePointerInfo(Info.ptrVal, Info.offset), Info.align, + Info.flags, Info.size); } else if (!HasChain) { Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); } else if (!I.getType()->isVoidTy()) { @@ -5822,6 +5821,7 @@ case Intrinsic::prefetch: { SDValue Ops[5]; unsigned rw = cast(I.getArgOperand(1))->getZExtValue(); + auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore; Ops[0] = getRoot(); Ops[1] = getValue(I.getArgOperand(0)); Ops[2] = getValue(I.getArgOperand(1)); @@ -5832,9 +5832,7 @@ EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), 0, /* align */ - false, /* volatile */ - rw==0, /* read */ - rw==1)); /* write */ + Flags)); return nullptr; } case Intrinsic::lifetime_start: Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7394,9 +7394,8 @@ Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1); Info.offset = 0; Info.align = 0; - Info.vol = false; // volatile loads with NEON intrinsics not supported - Info.readMem = true; - Info.writeMem = false; + // volatile loads with NEON intrinsics not supported + Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::aarch64_neon_st2: @@ -7421,9 +7420,8 @@ Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1); Info.offset = 0; Info.align = 0; - Info.vol = false; // volatile stores with NEON intrinsics not supported - Info.readMem = false; - Info.writeMem = true; + // volatile stores with NEON intrinsics not supported + Info.flags = MachineMemOperand::MOStore; return true; } case Intrinsic::aarch64_ldaxr: @@ -7434,9 +7432,7 @@ Info.ptrVal = I.getArgOperand(0); Info.offset = 0; Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); - Info.vol = true; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; return true; } case Intrinsic::aarch64_stlxr: @@ -7447,9 +7443,7 @@ Info.ptrVal = I.getArgOperand(1); Info.offset = 0; Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); - Info.vol = true; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; return true; } case Intrinsic::aarch64_ldaxp: @@ -7459,9 +7453,7 @@ Info.ptrVal = I.getArgOperand(0); Info.offset = 0; Info.align = 16; - Info.vol = true; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; return true; case Intrinsic::aarch64_stlxp: case Intrinsic::aarch64_stxp: @@ -7470,9 +7462,7 @@ Info.ptrVal = I.getArgOperand(2); Info.offset = 0; Info.align = 16; - Info.vol = true; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; return true; default: break; Index: lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/SIISelLowering.cpp +++ lib/Target/AMDGPU/SIISelLowering.cpp @@ -566,11 +566,12 @@ Info.memVT = MVT::getVT(CI.getType()); Info.ptrVal = CI.getOperand(0); Info.align = 0; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; const ConstantInt *Vol = dyn_cast(CI.getOperand(4)); - Info.vol = !Vol || !Vol->isZero(); - Info.readMem = true; - Info.writeMem = true; + if (!Vol || !Vol->isZero()) + Info.flags |= MachineMemOperand::MOVolatile; + return true; } default: Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -13455,9 +13455,8 @@ Info.offset = 0; Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1); Info.align = cast(AlignArg)->getZExtValue(); - Info.vol = false; // volatile loads with NEON intrinsics not supported - Info.readMem = true; - Info.writeMem = false; + // volatile loads with NEON intrinsics not supported + Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::arm_neon_vst1: @@ -13482,9 +13481,8 @@ Info.offset = 0; Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1); Info.align = cast(AlignArg)->getZExtValue(); - Info.vol = false; // volatile stores with NEON intrinsics not supported - Info.readMem = false; - Info.writeMem = true; + // volatile stores with NEON intrinsics not supported + Info.flags = MachineMemOperand::MOStore; return true; } case Intrinsic::arm_ldaex: @@ -13496,9 +13494,7 @@ Info.ptrVal = I.getArgOperand(0); Info.offset = 0; Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); - Info.vol = true; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; return true; } case Intrinsic::arm_stlex: @@ -13510,9 +13506,7 @@ Info.ptrVal = I.getArgOperand(1); Info.offset = 0; Info.align = DL.getABITypeAlignment(PtrTy->getElementType()); - Info.vol = true; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; return true; } case Intrinsic::arm_stlexd: @@ -13522,9 +13516,7 @@ Info.ptrVal = I.getArgOperand(2); Info.offset = 0; Info.align = 8; - Info.vol = true; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile; return true; case Intrinsic::arm_ldaexd: @@ -13534,9 +13526,7 @@ Info.ptrVal = I.getArgOperand(0); Info.offset = 0; Info.align = 8; - Info.vol = true; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; return true; default: Index: lib/Target/NVPTX/NVPTXISelLowering.cpp =================================================================== --- lib/Target/NVPTX/NVPTXISelLowering.cpp +++ lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -1561,8 +1561,7 @@ Chain = DAG.getMemIntrinsicNode( Op, dl, DAG.getVTList(MVT::Other, MVT::Glue), StoreOperands, TheStoreType, MachinePointerInfo(), EltAlign, - /* Volatile */ false, /* ReadMem */ false, - /* WriteMem */ true, /* Size */ 0); + MachineMemOperand::MOStore); InFlag = Chain.getValue(1); // Cleanup. @@ -1623,8 +1622,7 @@ Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs, CopyParamOps, elemtype, MachinePointerInfo(), /* Align */ 0, - /* Volatile */ false, /* ReadMem */ false, - /* WriteMem */ true, /* Size */ 0); + MachineMemOperand::MOStore); InFlag = Chain.getValue(1); } @@ -1810,8 +1808,8 @@ DAG.getConstant(Offsets[VecIdx], dl, MVT::i32), InFlag}; SDValue RetVal = DAG.getMemIntrinsicNode( Op, dl, DAG.getVTList(LoadVTs), LoadOperands, TheLoadType, - MachinePointerInfo(), EltAlign, /* Volatile */ false, - /* ReadMem */ true, /* WriteMem */ false, /* Size */ 0); + MachinePointerInfo(), EltAlign, + MachineMemOperand::MOLoad); for (unsigned j = 0; j < NumElts; ++j) { SDValue Ret = RetVal.getValue(j); @@ -2596,8 +2594,7 @@ Chain = DAG.getMemIntrinsicNode(Op, dl, DAG.getVTList(MVT::Other), StoreOperands, TheStoreType, MachinePointerInfo(), /* Align */ 1, - /* Volatile */ false, /* ReadMem */ false, - /* WriteMem */ true, /* Size */ 0); + MachineMemOperand::MOStore); // Cleanup vector state. StoreOperands.clear(); } @@ -3328,8 +3325,9 @@ // in order to model data exchange with other threads, but perform no real // memory accesses. Info.memVT = MVT::i1; - Info.readMem = true; // Our result depends on other thread's arguments. - Info.writeMem = true; // Other threads depend on our thread's argument. + + // Our result depends on both our and other thread's arguments. + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; return true; case Intrinsic::nvvm_wmma_load_a_f16_col: case Intrinsic::nvvm_wmma_load_a_f16_row: @@ -3359,9 +3357,7 @@ Info.memVT = MVT::v8f16; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; } @@ -3382,9 +3378,7 @@ Info.memVT = MVT::v4f16; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; } @@ -3405,9 +3399,7 @@ Info.memVT = MVT::v8f32; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; } @@ -3428,9 +3420,7 @@ Info.memVT = MVT::v4f16; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore; Info.align = 16; return true; } @@ -3451,9 +3441,7 @@ Info.memVT = MVT::v8f32; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore; Info.align = 16; return true; } @@ -3490,9 +3478,7 @@ Info.memVT = getValueType(DL, I.getType()); Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; Info.align = 0; return true; } @@ -3510,9 +3496,7 @@ Info.memVT = getValueType(DL, I.getType()); Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = cast(I.getArgOperand(1))->getZExtValue(); return true; @@ -3531,9 +3515,7 @@ Info.memVT = getValueType(DL, I.getType()); Info.ptrVal = I.getArgOperand(0); Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = cast(I.getArgOperand(1))->getZExtValue(); return true; @@ -3599,9 +3581,7 @@ Info.memVT = MVT::v4f32; Info.ptrVal = nullptr; Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; @@ -3721,9 +3701,7 @@ Info.memVT = MVT::v4i32; Info.ptrVal = nullptr; Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; @@ -3776,9 +3754,7 @@ Info.memVT = MVT::i8; Info.ptrVal = nullptr; Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; @@ -3831,9 +3807,7 @@ Info.memVT = MVT::i16; Info.ptrVal = nullptr; Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; @@ -3886,9 +3860,7 @@ Info.memVT = MVT::i32; Info.ptrVal = nullptr; Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; @@ -3926,9 +3898,7 @@ Info.memVT = MVT::i64; Info.ptrVal = nullptr; Info.offset = 0; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; Info.align = 16; return true; } Index: lib/Target/PowerPC/PPCISelLowering.cpp =================================================================== --- lib/Target/PowerPC/PPCISelLowering.cpp +++ lib/Target/PowerPC/PPCISelLowering.cpp @@ -2428,8 +2428,8 @@ SDValue Ops[] = { GA, Reg }; return DAG.getMemIntrinsicNode( PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0, false, true, - false, 0); + MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0, + MachineMemOperand::MOLoad); } SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, @@ -13348,9 +13348,7 @@ Info.offset = -VT.getStoreSize()+1; Info.size = 2*VT.getStoreSize()-1; Info.align = 1; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::ppc_qpx_qvlfda: @@ -13384,9 +13382,7 @@ Info.offset = 0; Info.size = VT.getStoreSize(); Info.align = 1; - Info.vol = false; - Info.readMem = true; - Info.writeMem = false; + Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::ppc_qpx_qvstfd: @@ -13438,9 +13434,7 @@ Info.offset = -VT.getStoreSize()+1; Info.size = 2*VT.getStoreSize()-1; Info.align = 1; - Info.vol = false; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore; return true; } case Intrinsic::ppc_qpx_qvstfda: @@ -13473,9 +13467,7 @@ Info.offset = 0; Info.size = VT.getStoreSize(); Info.align = 1; - Info.vol = false; - Info.readMem = false; - Info.writeMem = true; + Info.flags = MachineMemOperand::MOStore; return true; } default: Index: lib/Target/X86/X86ISelLowering.cpp =================================================================== --- lib/Target/X86/X86ISelLowering.cpp +++ lib/Target/X86/X86ISelLowering.cpp @@ -4494,9 +4494,7 @@ return false; Info.opc = ISD::INTRINSIC_W_CHAIN; - Info.readMem = false; - Info.writeMem = false; - Info.vol = false; + Info.flags = MachineMemOperand::MONone; Info.offset = 0; switch (IntrData->Type) { @@ -4504,14 +4502,14 @@ Info.ptrVal = I.getArgOperand(0); Info.memVT = MVT::getVT(I.getType()); Info.align = 1; - Info.readMem = true; + Info.flags |= MachineMemOperand::MOLoad; break; } case COMPRESS_TO_MEM: { Info.ptrVal = I.getArgOperand(0); Info.memVT = MVT::getVT(I.getArgOperand(1)->getType()); Info.align = 1; - Info.writeMem = true; + Info.flags |= MachineMemOperand::MOStore; break; } case TRUNCATE_TO_MEM_VI8: @@ -4529,7 +4527,7 @@ Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements()); Info.align = 1; - Info.writeMem = true; + Info.flags |= MachineMemOperand::MOStore; break; } default: @@ -6649,8 +6647,7 @@ DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT, LDBase->getPointerInfo(), LDBase->getAlignment(), - false/*isVolatile*/, true/*ReadMem*/, - false/*WriteMem*/); + MachineMemOperand::MOLoad); for (auto *LD : Loads) DAG.makeEquivalentMemoryOrdering(LD, ResNode); return DAG.getBitcast(VT, ResNode); @@ -19337,13 +19334,12 @@ DAG.getConstant(ArgMode, dl, MVT::i8), DAG.getConstant(Align, dl, MVT::i32)}; SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other); - SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl, - VTs, InstOps, MVT::i64, - MachinePointerInfo(SV), - /*Align=*/0, - /*Volatile=*/false, - /*ReadMem=*/true, - /*WriteMem=*/true); + SDValue VAARG = DAG.getMemIntrinsicNode( + X86ISD::VAARG_64, dl, + VTs, InstOps, MVT::i64, + MachinePointerInfo(SV), + /*Align=*/0, + MachineMemOperand::MOLoad | MachineMemOperand::MOStore); Chain = VAARG.getValue(1); // Load the next argument and return it