diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -2224,6 +2224,59 @@ if (Instruction *R = foldSelectGEP(GEP, Builder)) return R; + // The definition of llvm.ptrmask(Ptr, Mask) is: + // (getelementptr i8, Ptr, (sub (and (ptrtoint Ptr), Mask), (ptrtoint Ptr))) + // + // llvm.ptrmask is preferable as it doesn't require transitioning from ptr -> + // int (and is generally easier to analyze). + // Don't do this for for vector of pointers, as llvm.ptrmask has weird + // definition for vectors. + if (!GEPType->isVectorTy() && GEPEltType->isIntegerTy(8) && + GEP.getNumIndices() == 1) { + Value *Index = GEP.idx_begin()->get(); + if (Index->hasOneUse()) { + Value *Ptr = GEP.getPointerOperand(); + Value *Mask; + const APInt *CMask, *NotCMask; + Value *R = nullptr; + // (getelementptr i8, Ptr, (sub (and (ptrtoint Ptr), Mask), (ptrtoint Ptr))) + // -> llvm.ptrmask(Ptr, Mask) + if (match(Index, m_Sub(m_OneUse(m_c_And(m_PtrToInt(m_Specific(Ptr)), + m_Value(Mask))), + m_PtrToInt(m_Specific(Ptr))))) + R = Builder.CreateIntrinsic(Ptr->getType(), Intrinsic::ptrmask, + {Ptr, Mask}); + // (getelementptr i8, Ptr (sub 0, (and (ptrtoint Ptr), CMask))) + // -> llvm.ptrmask(Ptr, ~CMask) + // + // This is what __builtin_align_down generates. + else if (match(Index, m_Neg(m_OneUse(m_And(m_PtrToInt(m_Specific(Ptr)), + m_APInt(CMask))))) && + CMask->isMask()) + R = Builder.CreateIntrinsic( + Ptr->getType(), Intrinsic::ptrmask, + {Ptr, ConstantInt::get(Index->getType(), ~(*CMask))}); + + // (getelementptr i8, Ptr (sub (and (add (ptrtoint Ptr), ~CMask), CMask), (ptrtoint Ptr))) + // -> llvm.ptrmask((getelementptr i8, Ptr, ~CMask), CMask) + // + // This is what __builtin_align_up generates. + else if (match(Index, + m_Sub(m_OneUse( + m_And(m_OneUse(m_Add(m_PtrToInt(m_Specific(Ptr)), + m_APInt(NotCMask))), + m_APInt(CMask))), + m_PtrToInt(m_Specific(Ptr)))) && + (*CMask) == ~(*NotCMask) && NotCMask->isMask()) + R = Builder.CreateIntrinsic( + Ptr->getType(), Intrinsic::ptrmask, + {Builder.CreateGEP(GEP.getSourceElementType(), Ptr, + ConstantInt::get(Index->getType(), *NotCMask)), + ConstantInt::get(Index->getType(), *CMask)}); + if (R) + return replaceInstUsesWith(GEP, R); + } + } return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/to-ptrmask.ll b/llvm/test/Transforms/InstCombine/to-ptrmask.ll --- a/llvm/test/Transforms/InstCombine/to-ptrmask.ll +++ b/llvm/test/Transforms/InstCombine/to-ptrmask.ll @@ -7,10 +7,7 @@ define ptr @GEP_default_pattern_to_ptrmask(ptr %p, i64 %m) { ; CHECK-LABEL: define ptr @GEP_default_pattern_to_ptrmask ; CHECK-SAME: (ptr [[P:%.*]], i64 [[M:%.*]]) { -; CHECK-NEXT: [[PI:%.*]] = ptrtoint ptr [[P]] to i64 -; CHECK-NEXT: [[PI_M:%.*]] = and i64 [[PI]], [[M]] -; CHECK-NEXT: [[PI_OFF:%.*]] = sub i64 [[PI_M]], [[PI]] -; CHECK-NEXT: [[PM:%.*]] = getelementptr i8, ptr [[P]], i64 [[PI_OFF]] +; CHECK-NEXT: [[PM:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P]], i64 [[M]]) ; CHECK-NEXT: ret ptr [[PM]] ; %pi = ptrtoint ptr %p to i64 @@ -92,11 +89,8 @@ ; CHECK-LABEL: define ptr @builtin_align_up_to_ptrmask ; CHECK-SAME: (ptr [[X:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[X]] to i64 -; CHECK-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR]], 7 -; CHECK-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[OVER_BOUNDARY]], -8 -; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]] -; CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[DIFF]] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[X]], i64 7 +; CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[TMP0]], i64 -8) ; CHECK-NEXT: ret ptr [[ALIGNED_RESULT]] ; entry: @@ -174,10 +168,7 @@ ; CHECK-LABEL: define ptr @builtin_align_down_to_ptrmask ; CHECK-SAME: (ptr [[X:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[X]] to i64 -; CHECK-NEXT: [[AND:%.*]] = and i64 [[INTPTR]], 7 -; CHECK-NEXT: [[DIFF:%.*]] = sub nsw i64 0, [[AND]] -; CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[DIFF]] +; CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[X]], i64 -8) ; CHECK-NEXT: ret ptr [[ALIGNED_RESULT]] ; entry: