diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h --- a/llvm/include/llvm/Analysis/TargetFolder.h +++ b/llvm/include/llvm/Analysis/TargetFolder.h @@ -116,6 +116,9 @@ Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef IdxList, bool IsInBounds = false) const override { + if (!ConstantExpr::isSupportedGetElementPtr(Ty)) + return nullptr; + if (auto *PC = dyn_cast(Ptr)) { // Every index must be constant. if (any_of(IdxList, [](Value *V) { return !isa(V); })) diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h --- a/llvm/include/llvm/IR/ConstantFolder.h +++ b/llvm/include/llvm/IR/ConstantFolder.h @@ -105,6 +105,9 @@ Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef IdxList, bool IsInBounds = false) const override { + if (!ConstantExpr::isSupportedGetElementPtr(Ty)) + return nullptr; + if (auto *PC = dyn_cast(Ptr)) { // Every index must be constant. if (any_of(IdxList, [](Value *V) { return !isa(V); })) diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -1349,6 +1349,12 @@ /// supported. static bool isSupportedBinOp(unsigned Opcode); + /// Whether creating a constant expression for this getelementptr type is + /// supported. + static bool isSupportedGetElementPtr(const Type *SrcElemTy) { + return !SrcElemTy->isScalableTy(); + } + /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { return V->getValueID() == ConstantExprVal; diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -1050,11 +1050,15 @@ return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL); if (auto *GEP = dyn_cast(InstOrCE)) { + Type *SrcElemTy = GEP->getSourceElementType(); + if (!ConstantExpr::isSupportedGetElementPtr(SrcElemTy)) + return nullptr; + if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI)) return C; - return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0], - Ops.slice(1), GEP->isInBounds(), + return ConstantExpr::getGetElementPtr(SrcElemTy, Ops[0], Ops.slice(1), + GEP->isInBounds(), GEP->getInRangeIndex()); } diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -4916,7 +4916,7 @@ } // Check to see if this is constant foldable. - if (!isa(Ptr) || + if (!isa(Ptr) || !ConstantExpr::isSupportedGetElementPtr(SrcTy) || !all_of(Indices, [](Value *V) { return isa(V); })) return nullptr; diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1399,7 +1399,9 @@ return TypeID; } -static bool isConstExprSupported(uint8_t Opcode) { +static bool isConstExprSupported(const BitcodeConstant *BC) { + uint8_t Opcode = BC->Opcode; + // These are not real constant expressions, always consider them supported. if (Opcode >= BitcodeConstant::FirstSpecialOpcode) return true; @@ -1412,6 +1414,9 @@ if (Instruction::isBinaryOp(Opcode)) return ConstantExpr::isSupportedBinOp(Opcode); + if (Opcode == Instruction::GetElementPtr) + return ConstantExpr::isSupportedGetElementPtr(BC->SrcElemTy); + return Opcode != Instruction::FNeg; } @@ -1467,7 +1472,7 @@ ConstOps.push_back(C); // Materialize as constant expression if possible. - if (isConstExprSupported(BC->Opcode) && ConstOps.size() == Ops.size()) { + if (isConstExprSupported(BC) && ConstOps.size() == Ops.size()) { Constant *C; if (Instruction::isCast(BC->Opcode)) { C = UpgradeBitCastExpr(BC->Opcode, ConstOps[0], BC->getType()); diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -2349,24 +2349,6 @@ case Intrinsic::dbg_assign: case Intrinsic::dbg_value: return fixupDbgValue(II); - case Intrinsic::vscale: { - // If datalayout has no special restrictions on vector data layout, - // replace `llvm.vscale` by an equivalent constant expression - // to benefit from cheap constant propagation. - Type *ScalableVectorTy = - VectorType::get(Type::getInt8Ty(II->getContext()), 1, true); - if (DL->getTypeAllocSize(ScalableVectorTy).getKnownMinValue() == 8) { - auto *Null = Constant::getNullValue(ScalableVectorTy->getPointerTo()); - auto *One = ConstantInt::getSigned(II->getType(), 1); - auto *CGep = - ConstantExpr::getGetElementPtr(ScalableVectorTy, Null, One); - replaceAllUsesWith(II, ConstantExpr::getPtrToInt(CGep, II->getType()), - FreshBBs, IsHugeFunc); - II->eraseFromParent(); - return true; - } - break; - } case Intrinsic::masked_gather: return optimizeGatherScatterInst(II, II->getArgOperand(0)); case Intrinsic::masked_scatter: diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -2464,6 +2464,7 @@ Type *OnlyIfReducedTy) { PointerType *OrigPtrTy = cast(C->getType()->getScalarType()); assert(Ty && "Must specify element type"); + assert(isSupportedGetElementPtr(Ty) && "Element type is unsupported!"); assert(OrigPtrTy->isOpaqueOrPointeeTypeMatches(Ty)); if (Constant *FC = diff --git a/llvm/test/Bitcode/constexpr-scalable-vector-gep.ll b/llvm/test/Bitcode/constexpr-scalable-vector-gep.ll --- a/llvm/test/Bitcode/constexpr-scalable-vector-gep.ll +++ b/llvm/test/Bitcode/constexpr-scalable-vector-gep.ll @@ -1,5 +1,4 @@ -; RUN: llvm-as < %s | llvm-dis -expand-constant-exprs | FileCheck %s -; RUN: llvm-dis -expand-constant-exprs < %s.bc | FileCheck %s +; RUN: llvm-dis < %s.bc | FileCheck %s @g = extern_weak global [32 x i8] diff --git a/llvm/test/CodeGen/AArch64/sve-vscale.ll b/llvm/test/CodeGen/AArch64/sve-vscale.ll --- a/llvm/test/CodeGen/AArch64/sve-vscale.ll +++ b/llvm/test/CodeGen/AArch64/sve-vscale.ll @@ -46,7 +46,10 @@ ; CHECK: rdvl x0, #1 ; CHECK-NEXT: ret define i32 @rdvl_const() nounwind { - ret i32 mul nsw (i32 ptrtoint (* getelementptr (, * null, i64 1) to i32), i32 16) + %vscale.ptr = getelementptr , ptr null, i64 1 + %vscale.int = ptrtoint ptr %vscale.ptr to i32 + %vscale.scaled = mul nsw i32 %vscale.int, 16 + ret i32 %vscale.scaled } define i32 @vscale_1() nounwind { diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll --- a/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll +++ b/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll @@ -202,19 +202,6 @@ ret %i2 } -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Memory Access and Addressing Operations -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -define @load() { -; CHECK-LABEL: @load( -; CHECK-NEXT: [[R:%.*]] = load , ptr getelementptr (, ptr null, i64 1), align 16 -; CHECK-NEXT: ret [[R]] -; - %r = load , ptr getelementptr (, ptr null, i64 1) - ret %r -} - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Conversion Operations ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/vscale.ll b/llvm/test/Transforms/InstSimplify/ConstProp/vscale.ll --- a/llvm/test/Transforms/InstSimplify/ConstProp/vscale.ll +++ b/llvm/test/Transforms/InstSimplify/ConstProp/vscale.ll @@ -202,19 +202,6 @@ ret %i2 } -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Memory Access and Addressing Operations -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -define @load() { -; CHECK-LABEL: @load( -; CHECK-NEXT: [[R:%.*]] = load , ptr getelementptr (, ptr null, i64 1), align 16 -; CHECK-NEXT: ret [[R]] -; - %r = load , ptr getelementptr (, ptr null, i64 1) - ret %r -} - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Conversion Operations ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll --- a/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll +++ b/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll @@ -152,51 +152,6 @@ ;; Memory Access and Addressing Operations ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; getelementptr - -define @getelementptr_constant_foldable_1() { -; CHECK-LABEL: @getelementptr_constant_foldable_1( -; CHECK-NEXT: ret zeroinitializer -; - %ptr = getelementptr i32, zeroinitializer, undef - ret %ptr -} - -define @getelementptr_constant_foldable_2() { -; CHECK-LABEL: @getelementptr_constant_foldable_2( -; CHECK-NEXT: ret zeroinitializer -; - %ptr = getelementptr , ptr null, undef - ret %ptr -} - -; fold getelementptr P, 0 -> P. -define ptr @getelementptr_constant_foldable_3() { -; CHECK-LABEL: @getelementptr_constant_foldable_3( -; CHECK-NEXT: ret ptr null -; - ret ptr null -} - -define ptr @getelementptr_not_constant_foldable(i64 %x) { -; CHECK-LABEL: @getelementptr_not_constant_foldable( -; CHECK-NEXT: [[PTR:%.*]] = getelementptr , ptr null, i64 [[X:%.*]] -; CHECK-NEXT: ret ptr [[PTR]] -; - %ptr = getelementptr , ptr null, i64 %x - ret ptr %ptr -} - -; Check GEP's result is known to be non-null. -define i1 @getelementptr_check_non_null(ptr %ptr) { -; CHECK-LABEL: @getelementptr_check_non_null( -; CHECK-NEXT: ret i1 false -; - %x = getelementptr inbounds , ptr %ptr, i32 1 - %cmp = icmp eq ptr %x, null - ret i1 %cmp -} - define i32 @extractelement_splat_constant_index(i32 %v) { ; CHECK-LABEL: @extractelement_splat_constant_index( ; CHECK-NEXT: ret i32 [[V:%.*]] diff --git a/llvm/test/Transforms/InstSimplify/vscale.ll b/llvm/test/Transforms/InstSimplify/vscale.ll --- a/llvm/test/Transforms/InstSimplify/vscale.ll +++ b/llvm/test/Transforms/InstSimplify/vscale.ll @@ -174,28 +174,29 @@ ret %ptr } -define @getelementptr_constant_foldable_2() { +; fold getelementptr P, 0 -> P. +define ptr @getelementptr_constant_foldable_2() { ; CHECK-LABEL: @getelementptr_constant_foldable_2( -; CHECK-NEXT: ret zeroinitializer +; CHECK-NEXT: ret ptr null ; - %ptr = getelementptr , ptr null, undef - ret %ptr + ret ptr null } -; fold getelementptr P, 0 -> P. -define ptr @getelementptr_constant_foldable_3() { -; CHECK-LABEL: @getelementptr_constant_foldable_3( -; CHECK-NEXT: ret ptr null +define @getelementptr_not_constant_foldable_1() { +; CHECK-LABEL: @getelementptr_not_constant_foldable_1( +; CHECK-NEXT: [[PTR:%.*]] = getelementptr , ptr null, undef +; CHECK-NEXT: ret [[PTR]] ; - ret ptr null + %ptr = getelementptr , ptr null, undef + ret %ptr } -define ptr @getelementptr_not_constant_foldable(i64 %x) { -; CHECK-LABEL: @getelementptr_not_constant_foldable( -; CHECK-NEXT: [[PTR:%.*]] = getelementptr , ptr null, i64 [[X:%.*]] +define ptr @getelementptr_not_constant_foldable_2(i64 %x) { +; CHECK-LABEL: @getelementptr_not_constant_foldable_2( +; CHECK-NEXT: [[PTR:%.*]] = getelementptr , ptr null, i64 1 ; CHECK-NEXT: ret ptr [[PTR]] ; - %ptr = getelementptr , ptr null, i64 %x + %ptr = getelementptr , ptr null, i64 1 ret ptr %ptr }