[LLVM] Remove support for constant scalable vector GEPs.

[paulwalker-arm]

Summary

This work has fallen out from D134648 as a requirement to loosen
the "constness" of vscale.

[paulwalker-arm]

Tighten up a couple of function signatures.

[paulwalker-arm]

llvm/lib/CodeGen/CodeGenPrepare.cpp

2353–2355 ↗

(On Diff #502732)

Just being able to remove this wholesale seems optimistic but it doesn't seem to have any relevant test coverage.

llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll

157–160

These tests look like blind copies from vscale.ll and have no insert_eltement_poison related requirements so I've just remove them.

[nikic]

llvm/lib/CodeGen/CodeGenPrepare.cpp

2353–2355 ↗

(On Diff #502732)

I'd suggest landing this as a separate change ahead of time.

llvm/test/Transforms/InstSimplify/vscale.ll

190

This case should still fold.

[paulwalker-arm]

Restored simplification for case where isSupportedGetElementPtr() is false.

[nikic]

LGTM. This is slightly non-optimal in terms of constant folding (what we really want is to export a GEP-folding API in ConstantFolding and not force going through ConstExpr + ConstantFoldConstant), but I don't think we need to block on that.

llvm/test/Transforms/InstSimplify/vscale.ll

198

Any reason for this change?

[nikic]

This should also get a LangRef mention that vscale is not valid in constant expression GEPs.

[paulwalker-arm]

llvm/test/Transforms/InstSimplify/vscale.ll

198

I wanted to show that even though all operands are constant this form of GEP is still not constant foldable. I figured this to be a better version of the original test, but I might have missed some subtlety that means an extra test is more appropriate?

[nikic]

llvm/test/Transforms/InstSimplify/vscale.ll

198

Oh I see, that makes sense.

[paulwalker-arm]

llvm/lib/CodeGen/CodeGenPrepare.cpp

2353–2355 ↗

(On Diff #502732)

Just a note to say I've followed this advice and landed the CodeGenPrepare change via https://reviews.llvm.org/rGadbdf273efd5 and will await any fallout before landing the rest of this patch, probably towards the end of next week.

llvm/include/llvm/Analysis/TargetFolder.h

Value *FoldUnOpFMF(Instruction::UnaryOps Opc, Value *V,
                       FastMathFlags FMF) const override {
     if (Constant *C = dyn_cast<Constant>(V))
       return ConstantFoldUnaryOpOperand(Opc, C, DL);
     return nullptr;
   }
 
   Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
                  bool IsInBounds = false) const override {
+    if (!ConstantExpr::isSupportedGetElementPtr(Ty))
+      return nullptr;
+
     if (auto *PC = dyn_cast<Constant>(Ptr)) {
       // Every index must be constant.
       if (any_of(IdxList, [](Value *V) { return !isa<Constant>(V); }))
         return nullptr;
       if (IsInBounds)
         return Fold(ConstantExpr::getInBoundsGetElementPtr(Ty, PC, IdxList));
       else
         return Fold(ConstantExpr::getGetElementPtr(Ty, PC, IdxList));

llvm/include/llvm/IR/ConstantFolder.h

Value *FoldICmp(CmpInst::Predicate P, Value *LHS, Value *RHS) const override {
     auto *RC = dyn_cast<Constant>(RHS);
     if (LC && RC)
       return ConstantExpr::getCompare(P, LC, RC);
     return nullptr;
   }
 
   Value *FoldGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList,
                  bool IsInBounds = false) const override {
+    if (!ConstantExpr::isSupportedGetElementPtr(Ty))
+      return nullptr;
+
     if (auto *PC = dyn_cast<Constant>(Ptr)) {
       // Every index must be constant.
       if (any_of(IdxList, [](Value *V) { return !isa<Constant>(V); }))
         return nullptr;
 
       if (IsInBounds)
         return ConstantExpr::getInBoundsGetElementPtr(Ty, PC, IdxList);
       else

llvm/include/llvm/IR/Constants.h

public:
   /// Whether creating a constant expression for this binary operator is
   /// desirable.
   static bool isDesirableBinOp(unsigned Opcode);
 
   /// Whether creating a constant expression for this binary operator is
   /// supported.
   static bool isSupportedBinOp(unsigned Opcode);
 
+  /// Whether creating a constant expression for this getelementptr type is
+  /// supported.
+  static bool isSupportedGetElementPtr(const Type *SrcElemTy) {
+    return !SrcElemTy->isScalableTy();
+  }
+
   /// Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Value *V) {
     return V->getValueID() == ConstantExprVal;
   }
 
 private:
   // Shadow Value::setValueSubclassData with a private forwarding method so that
   // subclasses cannot accidentally use it.

llvm/lib/Analysis/ConstantFolding.cpp

if (Instruction::isBinaryOp(Opcode)) {
     }
     return ConstantFoldBinaryOpOperands(Opcode, Ops[0], Ops[1], DL);
   }
 
   if (Instruction::isCast(Opcode))
     return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL);
 
   if (auto *GEP = dyn_cast<GEPOperator>(InstOrCE)) {
+    Type *SrcElemTy = GEP->getSourceElementType();
+    if (!ConstantExpr::isSupportedGetElementPtr(SrcElemTy))
+      return nullptr;
+
     if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI))
       return C;
 
-    return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0],
-                                          Ops.slice(1), GEP->isInBounds(),
+    return ConstantExpr::getGetElementPtr(SrcElemTy, Ops[0], Ops.slice(1),
+                                          GEP->isInBounds(),
                                           GEP->getInRangeIndex());
   }
 
   if (auto *CE = dyn_cast<ConstantExpr>(InstOrCE)) {
     if (CE->isCompare())
       return ConstantFoldCompareInstOperands(CE->getPredicate(), Ops[0], Ops[1],
                                              DL, TLI);
     return CE->getWithOperands(Ops);

llvm/lib/Analysis/InstructionSimplify.cpp

if (!IsScalableVec && Q.DL.getTypeAllocSize(LastType) == 1 &&
     }
   }
 
   // Check to see if this is constant foldable.
   if (!isa<Constant>(Ptr) ||
       !all_of(Indices, [](Value *V) { return isa<Constant>(V); }))
     return nullptr;
 
+  if (!ConstantExpr::isSupportedGetElementPtr(SrcTy))
+    return ConstantFoldGetElementPtr(SrcTy, cast<Constant>(Ptr), InBounds,
+                                     std::nullopt, Indices);
+
   auto *CE = ConstantExpr::getGetElementPtr(SrcTy, cast<Constant>(Ptr), Indices,
                                             InBounds);
   return ConstantFoldConstant(CE, Q.DL);
 }
 
 Value *llvm::simplifyGEPInst(Type *SrcTy, Value *Ptr, ArrayRef<Value *> Indices,
                              bool InBounds, const SimplifyQuery &Q) {
   return ::simplifyGEPInst(SrcTy, Ptr, Indices, InBounds, Q, RecursionLimit);

llvm/lib/Bitcode/Reader/BitcodeReader.cpp

#endif
   unsigned TypeID = TypeList.size();
   TypeList.push_back(Ty);
   if (!ChildTypeIDs.empty())
     append_range(ContainedTypeIDs[TypeID], ChildTypeIDs);
   VirtualTypeIDs.insert({CacheKey, TypeID});
   return TypeID;
 }
 
-static bool isConstExprSupported(uint8_t Opcode) {
+static bool isConstExprSupported(const BitcodeConstant *BC) {
+  uint8_t Opcode = BC->Opcode;
+
   // These are not real constant expressions, always consider them supported.
   if (Opcode >= BitcodeConstant::FirstSpecialOpcode)
     return true;
 
   // If -expand-constant-exprs is set, we want to consider all expressions
   // as unsupported.
   if (ExpandConstantExprs)
     return false;
 
   if (Instruction::isBinaryOp(Opcode))
     return ConstantExpr::isSupportedBinOp(Opcode);
 
+  if (Opcode == Instruction::GetElementPtr)
+    return ConstantExpr::isSupportedGetElementPtr(BC->SrcElemTy);
+
   return Opcode != Instruction::FNeg;
 }
 
 Expected<Value *> BitcodeReader::materializeValue(unsigned StartValID,
                                                   BasicBlock *InsertBB) {
   // Quickly handle the case where there is no BitcodeConstant to resolve.
   if (StartValID < ValueList.size() && ValueList[StartValID] &&
       !isa<BitcodeConstant>(ValueList[StartValID]))
while (!Worklist.empty()) {
     std::reverse(Ops.begin(), Ops.end());
 
     SmallVector<Constant *> ConstOps;
     for (Value *Op : Ops)
       if (auto *C = dyn_cast<Constant>(Op))
         ConstOps.push_back(C);
 
     // Materialize as constant expression if possible.
-    if (isConstExprSupported(BC->Opcode) && ConstOps.size() == Ops.size()) {
+    if (isConstExprSupported(BC) && ConstOps.size() == Ops.size()) {
       Constant *C;
       if (Instruction::isCast(BC->Opcode)) {
         C = UpgradeBitCastExpr(BC->Opcode, ConstOps[0], BC->getType());
         if (!C)
           C = ConstantExpr::getCast(BC->Opcode, ConstOps[0], BC->getType());
       } else if (Instruction::isBinaryOp(BC->Opcode)) {
         C = ConstantExpr::get(BC->Opcode, ConstOps[0], ConstOps[1], BC->Flags);
       } else {

llvm/lib/IR/Constants.cpp

 }
 
 Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
                                          ArrayRef<Value *> Idxs, bool InBounds,
                                          std::optional<unsigned> InRangeIndex,
                                          Type *OnlyIfReducedTy) {
   PointerType *OrigPtrTy = cast<PointerType>(C->getType()->getScalarType());
   assert(Ty && "Must specify element type");
+  assert(isSupportedGetElementPtr(Ty) && "Element type is unsupported!");
   assert(OrigPtrTy->isOpaqueOrPointeeTypeMatches(Ty));
 
   if (Constant *FC =
           ConstantFoldGetElementPtr(Ty, C, InBounds, InRangeIndex, Idxs))
     return FC;          // Fold a few common cases.
 
   // Get the result type of the getelementptr!
   Type *DestTy = GetElementPtrInst::getIndexedType(Ty, Idxs);

llvm/test/Bitcode/constexpr-scalable-vector-gep.ll

-; RUN: llvm-as < %s | llvm-dis -expand-constant-exprs | FileCheck %s
-; RUN: llvm-dis -expand-constant-exprs < %s.bc | FileCheck %s
+; RUN: llvm-dis < %s.bc | FileCheck %s
 
 @g = extern_weak global [32 x i8]
 
 define ptr @test_scalable_vector_gep() {
 ; CHECK-LABEL: define ptr @test_scalable_vector_gep() {
 ; CHECK-NEXT: %constexpr = getelementptr <vscale x 1 x i8>, ptr @g, i64 1
 ; CHECK-NEXT: ret ptr %constexpr
   ret ptr getelementptr (<vscale x 1 x i8>, ptr @g, i64 1)
 }

llvm/test/CodeGen/AArch64/sve-vscale.ll

define i64 @rdvl_i64() nounwind {
   %1 = mul nsw i64 %vscale, 16
   ret i64 %1
 }
 
 ; CHECK-LABEL: rdvl_const:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
 define i32 @rdvl_const() nounwind {
-  ret i32 mul nsw (i32 ptrtoint (<vscale x 1 x i8>* getelementptr (<vscale x 1 x i8>, <vscale x 1 x i8>* null, i64 1) to i32), i32 16)
+  %vscale.ptr = getelementptr <vscale x 1 x i8>, ptr null, i64 1
+  %vscale.int = ptrtoint ptr %vscale.ptr to i32
+  %vscale.scaled = mul nsw i32 %vscale.int, 16
+  ret i32 %vscale.scaled
 }
 
 define i32 @vscale_1() nounwind {
 ; CHECK-LABEL: vscale_1:
 ; CHECK:       rdvl [[TMP:x[0-9]+]], #1
 ; CHECK-NEXT:  lsr  x0, [[TMP]], #4
 ; CHECK-NEXT:  ret
   %vscale = call i32 @llvm.vscale.i32()

llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll

 ; CHECK-NEXT:    ret <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
 ;
   %i = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
   %i2 = shufflevector <vscale x 4 x i32> %i, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
   ret <vscale x 4 x i32> %i2
 }
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Memory Access and Addressing Operations
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-define <vscale x 2 x double> @load() {
-; CHECK-LABEL: @load(
-; CHECK-NEXT:    [[R:%.*]] = load <vscale x 2 x double>, ptr getelementptr (<vscale x 2 x double>, ptr null, i64 1), align 16
-; CHECK-NEXT:    ret <vscale x 2 x double> [[R]]
-;
-  %r = load <vscale x 2 x double>, ptr getelementptr (<vscale x 2 x double>, ptr null, i64 1)
-  ret <vscale x 2 x double> %r
-}
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Conversion Operations
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 define <vscale x 4 x float> @bitcast() {
 ; CHECK-LABEL: @bitcast(
 ; CHECK-NEXT:    ret <vscale x 4 x float> bitcast (<vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer) to <vscale x 4 x float>)
 ;
   %i1 = insertelement <vscale x 4 x i32> poison, i32 1, i32 0

llvm/test/Transforms/InstSimplify/ConstProp/vscale.ll

 ; CHECK-NEXT:    ret <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
 ;
   %i = insertelement <vscale x 4 x i32> undef, i32 1, i32 0
   %i2 = shufflevector <vscale x 4 x i32> %i, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
   ret <vscale x 4 x i32> %i2
 }
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Memory Access and Addressing Operations
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-define <vscale x 2 x double> @load() {
-; CHECK-LABEL: @load(
-; CHECK-NEXT:    [[R:%.*]] = load <vscale x 2 x double>, ptr getelementptr (<vscale x 2 x double>, ptr null, i64 1), align 16
-; CHECK-NEXT:    ret <vscale x 2 x double> [[R]]
-;
-  %r = load <vscale x 2 x double>, ptr getelementptr (<vscale x 2 x double>, ptr null, i64 1)
-  ret <vscale x 2 x double> %r
-}
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Conversion Operations
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 define <vscale x 4 x float> @bitcast() {
 ; CHECK-LABEL: @bitcast(
 ; CHECK-NEXT:    ret <vscale x 4 x float> bitcast (<vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer) to <vscale x 4 x float>)
 ;
   %i1 = insertelement <vscale x 4 x i32> undef, i32 1, i32 0

llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll

;
   %i3 = bitcast <vscale x 4 x i32> %i2 to <vscale x 4 x float>
   ret <vscale x 4 x float> %i3
 }
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Memory Access and Addressing Operations
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-; getelementptr
-
-define <vscale x 4 x ptr> @getelementptr_constant_foldable_1() {
-; CHECK-LABEL: @getelementptr_constant_foldable_1(
-; CHECK-NEXT:    ret <vscale x 4 x ptr> zeroinitializer
-;
-  %ptr = getelementptr i32, <vscale x 4 x ptr> zeroinitializer, <vscale x 4 x i64> undef
-  ret <vscale x 4 x ptr> %ptr
-}
-
-define <vscale x 4 x ptr> @getelementptr_constant_foldable_2() {
-; CHECK-LABEL: @getelementptr_constant_foldable_2(
-; CHECK-NEXT:    ret <vscale x 4 x ptr> zeroinitializer
-;
-  %ptr = getelementptr <vscale x 4 x i32>, ptr null, <vscale x 4 x i64> undef
-  ret <vscale x 4 x ptr> %ptr
-}
-
-; fold getelementptr P, 0 -> P.
-define ptr @getelementptr_constant_foldable_3() {
-; CHECK-LABEL: @getelementptr_constant_foldable_3(
-; CHECK-NEXT:    ret ptr null
-;
-  ret ptr null
-}
-
-define ptr @getelementptr_not_constant_foldable(i64 %x) {
-; CHECK-LABEL: @getelementptr_not_constant_foldable(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr <vscale x 4 x i32>, ptr null, i64 [[X:%.*]]
-; CHECK-NEXT:    ret ptr [[PTR]]
-;
-  %ptr = getelementptr <vscale x 4 x i32>, ptr null, i64 %x
-  ret ptr %ptr
-}
-
-; Check GEP's result is known to be non-null.
-define i1 @getelementptr_check_non_null(ptr %ptr) {
-; CHECK-LABEL: @getelementptr_check_non_null(
-; CHECK-NEXT:    ret i1 false
-;
-  %x = getelementptr inbounds <vscale x 16 x i8>, ptr %ptr, i32 1
-  %cmp = icmp eq ptr %x, null
-  ret i1 %cmp
-}
-
 define i32 @extractelement_splat_constant_index(i32 %v) {
 ; CHECK-LABEL: @extractelement_splat_constant_index(
 ; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
   %r = extractelement <vscale x 4 x i32> %splat, i32 1
   ret i32 %r

llvm/test/Transforms/InstSimplify/vscale.ll

;
   ret <vscale x 4 x ptr> %ptr
 }
 
 ; fold getelementptr P, 0 -> P.
 define ptr @getelementptr_constant_foldable_3() {
 ; CHECK-LABEL: @getelementptr_constant_foldable_3(
 ; CHECK-NEXT:    ret ptr null
 ;
   ret ptr null
 }
 
 define ptr @getelementptr_not_constant_foldable(i64 %x) {
 ; CHECK-LABEL: @getelementptr_not_constant_foldable(
-; CHECK-NEXT:    [[PTR:%.*]] = getelementptr <vscale x 4 x i32>, ptr null, i64 [[X:%.*]]
+; CHECK-NEXT:    [[PTR:%.*]] = getelementptr <vscale x 4 x i32>, ptr null, i64 1
 ; CHECK-NEXT:    ret ptr [[PTR]]
 ;
-  %ptr = getelementptr <vscale x 4 x i32>, ptr null, i64 %x
+  %ptr = getelementptr <vscale x 4 x i32>, ptr null, i64 1
   ret ptr %ptr
 }
 
 ; Check GEP's result is known to be non-null.
 define i1 @getelementptr_check_non_null(ptr %ptr) {
 ; CHECK-LABEL: @getelementptr_check_non_null(
 ; CHECK-NEXT:    ret i1 false
 ;