[OpaquePtr] Return opaque pointer from opaque pointer GEP

[nikic]

Summary

For a GEP on an opaque pointer, also return an opaque pointer (or vector of opaque pointer) result.

This requires explicitly enumerating the GEP source element type, because it is now no longer implicitly enumerated as part of either the source or result pointer types.

[aeubanks]

lgtm

what about GEP constants? do we need to do something special for those in ValueEnumerator?

[nikic]

In D104652#2830796, @aeubanks wrote:

lgtm

what about GEP constants? do we need to do something special for those in ValueEnumerator?

Yeah, and also a few more changes. I'll create a followup patch.

llvm/include/llvm/IR/Instructions.h

public:
   unsigned getPointerAddressSpace() const {
     return getPointerOperandType()->getPointerAddressSpace();
   }
 
   /// Returns the pointer type returned by the GEP
   /// instruction, which may be a vector of pointers.
   static Type *getGEPReturnType(Type *ElTy, Value *Ptr,
                                 ArrayRef<Value *> IdxList) {
-    Type *PtrTy = PointerType::get(checkGEPType(getIndexedType(ElTy, IdxList)),
-                                   Ptr->getType()->getPointerAddressSpace());
+    PointerType *OrigPtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
+    unsigned AddrSpace = OrigPtrTy->getAddressSpace();
+    Type *ResultElemTy = checkGEPType(getIndexedType(ElTy, IdxList));
+    Type *PtrTy = OrigPtrTy->isOpaque()
+      ? PointerType::get(OrigPtrTy->getContext(), AddrSpace)
+      : PointerType::get(ResultElemTy, AddrSpace);
     // Vector GEP
     if (auto *PtrVTy = dyn_cast<VectorType>(Ptr->getType())) {
       ElementCount EltCount = PtrVTy->getElementCount();
       return VectorType::get(PtrTy, EltCount);
     }
     for (Value *Index : IdxList)
       if (auto *IndexVTy = dyn_cast<VectorType>(Index->getType())) {
         ElementCount EltCount = IndexVTy->getElementCount();

llvm/lib/Bitcode/Writer/ValueEnumerator.cpp

for (const BasicBlock &BB : F)
                 EnumerateMetadata(&F, VAM);
             continue;
           }
 
           EnumerateMetadata(&F, MD->getMetadata());
         }
         if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
           EnumerateType(SVI->getShuffleMaskForBitcode()->getType());
+        if (auto *GEP = dyn_cast<GetElementPtrInst>(&I))
+          EnumerateType(GEP->getSourceElementType());
         EnumerateType(I.getType());
         if (const auto *Call = dyn_cast<CallBase>(&I))
           EnumerateAttributes(Call->getAttributes());
 
         // Enumerate metadata attached with this instruction.
         MDs.clear();
         I.getAllMetadataOtherThanDebugLoc(MDs);
         for (unsigned i = 0, e = MDs.size(); i != e; ++i)

llvm/test/Assembler/opaque-ptr.ll

 ; CHECK: define void @store(ptr %a, i32 %i)
 ; CHECK:     store i32 %i, ptr %a
 ; CHECK:     ret void
 define void @store(ptr %a, i32 %i) {
     store i32 %i, ptr %a
     ret void
 }
 
-; CHECK: define void @gep(ptr %a)
-; CHECK:     %b = getelementptr i8, ptr %a, i32 2
-; CHECK:     ret void
-define void @gep(ptr %a) {
-    %b = getelementptr i8, ptr %a, i32 2
-    ret void
+; CHECK: define ptr @gep(ptr %a)
+; CHECK:     %res = getelementptr i8, ptr %a, i32 2
+; CHECK:     ret ptr %res
+define ptr @gep(ptr %a) {
+  %res = getelementptr i8, ptr %a, i32 2
+  ret ptr %res
+}
+
+; CHECK: define <2 x ptr> @gep_vec1(ptr %a)
+; CHECK:     %res = getelementptr i8, ptr %a, <2 x i32> <i32 1, i32 2>
+; CHECK:     ret <2 x ptr> %res
+define <2 x ptr> @gep_vec1(ptr %a) {
+  %res = getelementptr i8, ptr %a, <2 x i32> <i32 1, i32 2>
+  ret <2 x ptr> %res
+}
+
+; CHECK: define <2 x ptr> @gep_vec2(<2 x ptr> %a)
+; CHECK:     %res = getelementptr i8, <2 x ptr> %a, i32 2
+; CHECK:     ret <2 x ptr> %res
+define <2 x ptr> @gep_vec2(<2 x ptr> %a) {
+  %res = getelementptr i8, <2 x ptr> %a, i32 2
+  ret <2 x ptr> %res
 }
 
 ; CHECK: define void @cmpxchg(ptr %p, i32 %a, i32 %b)
 ; CHECK:     %val_success = cmpxchg ptr %p, i32 %a, i32 %b acq_rel monotonic
 ; CHECK:     ret void
 define void @cmpxchg(ptr %p, i32 %a, i32 %b) {
     %val_success = cmpxchg ptr %p, i32 %a, i32 %b acq_rel monotonic
     ret void