[ptr_provenance] Bitcode reader/writer support for the optional ptr_provenance operand of load/store instructions

[jeroen.dobbelaere]

Summary

This patch introduces bitcode reader/writer support for the optional ptr_provenance operand of load/store instructions.

This is done in a backwards and future compatible way: when present, an extra 'bool true' + the value and type are pushed.
When reading, if there are extra values, the first bool indicates if a ptr_provenance operand is available or not.

If in future more optional values are needed, extra bools can be added to indicate that they are present. As soon as one extra
optional value is emitted, the status of the others should also be emitted.

Note: this corresponds to D87254

[jeroen.dobbelaere]

• Rebased

[jeroen.dobbelaere]

Rebased

[jeroen.dobbelaere]

Rebase to 49cb4fef02e635bf304906232214166c7531d753

[jeroen.dobbelaere]

Rebased to ffe262a198a9f9030991df6d3ddd812e74fa3523

[jdoerfert]

Why do we add the boolean, isn't a nullptr/unknown provenance just as good?

[jeroen.dobbelaere]

In D104270#3883808, @jdoerfert wrote:

Why do we add the boolean, isn't a nullptr/unknown provenance just as good?

(I misinterpreted this as pushing an actual '0 pointer' to indicate that the ptr_provenance is not used. As far as I see, that is not possible.)
The boolean is to indicate that we do (or do not) have the optional ptr_provenance. It allows us to add more arguments to these instructions later (if that would be needed).
Just depending on the 'number of arguments' would not be sufficient here. So this is for forwards compatibility.

A nullptr, or unknown_provenance have their own meaning in the ptr_provenance world. At this point, a missing ptr_provenance argument indicates that the provenance is that of the pointer argument.
At a later time we can decide to move to a mechanism that the ptr_provenance should always be present and then the 'unknown_provenance' would be the best choice if its was missing.

[jdoerfert]

LG

[jeroen.dobbelaere]

Rebased to ef545ef62a833152d8975ff16333b57cc41befcc (Jan 9, 2023)

[jeroen.dobbelaere]

Rebased to: 8c7cfa357280dd93d33b10bbba0fe33797e27d63 (Feb 14, 2023)

llvm/lib/Bitcode/Reader/BitcodeReader.cpp

case bitc::FUNC_CODE_INST_ALLOCA: { // ALLOCA: [instty, opty, op, align]
       AllocaInst *AI = new AllocaInst(Ty, AS, Size, *Align);
       AI->setUsedWithInAlloca(InAlloca);
       AI->setSwiftError(SwiftError);
       I = AI;
       ResTypeID = getVirtualTypeID(AI->getType(), TyID);
       InstructionList.push_back(I);
       break;
     }
-    case bitc::FUNC_CODE_INST_LOAD: { // LOAD: [opty, op, align, vol]
+    case bitc::FUNC_CODE_INST_LOAD: {
+      // LOAD: [opty, op, align, vol, [hasPtrProv, [PtrProv]]]
       unsigned OpNum = 0;
       Value *Op;
       unsigned OpTypeID;
       if (getValueTypePair(Record, OpNum, NextValueNo, Op, OpTypeID, CurBB) ||
-          (OpNum + 2 != Record.size() && OpNum + 3 != Record.size()))
+          (OpNum + 2 > Record.size()))
         return error("Invalid record");
 
       if (!isa<PointerType>(Op->getType()))
         return error("Load operand is not a pointer type");
 
       Type *Ty = nullptr;
-      if (OpNum + 3 == Record.size()) {
+      if (OpNum + 3 <= Record.size()) {
         ResTypeID = Record[OpNum++];
         Ty = getTypeByID(ResTypeID);
       } else {
         ResTypeID = getContainedTypeID(OpTypeID);
         Ty = getTypeByID(ResTypeID);
         if (!Ty)
           return error("Missing element type for old-style load");
       }
 
       if (Error Err = typeCheckLoadStoreInst(Ty, Op->getType()))
         return Err;
 
       MaybeAlign Align;
       if (Error Err = parseAlignmentValue(Record[OpNum], Align))
         return Err;
       SmallPtrSet<Type *, 4> Visited;
       if (!Align && !Ty->isSized(&Visited))
         return error("load of unsized type");
       if (!Align)
         Align = TheModule->getDataLayout().getABITypeAlign(Ty);
-      I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align);
+      bool IsVolatile = Record[OpNum + 1];
+      OpNum += 2;
+      Value *PtrProvenance = nullptr;
+      if (OpNum < Record.size()) {
+        // Do we have a ptr_provenance ?
+        if (Record[OpNum++]) {
+          unsigned PtrProvenanceTypeID;
+          if (getValueTypePair(Record, OpNum, NextValueNo, PtrProvenance,
+                               PtrProvenanceTypeID, CurBB))
+            return error("Invalid ptr_provenance operand");
+        }
+      }
+      if (OpNum != Record.size())
+        return error("Invalid record");
+      I = new LoadInst(Ty, Op, "", IsVolatile, *Align);
+      if (PtrProvenance)
+        cast<LoadInst>(I)->setPtrProvenanceOperand(PtrProvenance);
       InstructionList.push_back(I);
       break;
     }
     case bitc::FUNC_CODE_INST_LOADATOMIC: {
-       // LOADATOMIC: [opty, op, align, vol, ordering, ssid]
+      // LOADATOMIC: [opty, op, align, vol, ordering, ssid,
+      //              [hasPtrProv,[PtrProv]]]
       unsigned OpNum = 0;
       Value *Op;
       unsigned OpTypeID;
       if (getValueTypePair(Record, OpNum, NextValueNo, Op, OpTypeID, CurBB) ||
-          (OpNum + 4 != Record.size() && OpNum + 5 != Record.size()))
+          (OpNum + 4 > Record.size()))
         return error("Invalid record");
 
       if (!isa<PointerType>(Op->getType()))
         return error("Load operand is not a pointer type");
 
       Type *Ty = nullptr;
-      if (OpNum + 5 == Record.size()) {
+      if (OpNum + 5 <= Record.size()) {
         ResTypeID = Record[OpNum++];
         Ty = getTypeByID(ResTypeID);
       } else {
         ResTypeID = getContainedTypeID(OpTypeID);
         Ty = getTypeByID(ResTypeID);
         if (!Ty)
           return error("Missing element type for old style atomic load");
       }
case bitc::FUNC_CODE_INST_LOADATOMIC: {
         return error("Invalid record");
       SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
 
       MaybeAlign Align;
       if (Error Err = parseAlignmentValue(Record[OpNum], Align))
         return Err;
       if (!Align)
         return error("Alignment missing from atomic load");
-      I = new LoadInst(Ty, Op, "", Record[OpNum + 1], *Align, Ordering, SSID);
+      bool IsVolatile = Record[OpNum + 1];
+      OpNum += 4;
+      Value *PtrProvenance = nullptr;
+      if (OpNum < Record.size()) {
+        // Do we have a ptr_provenance ?
+        if (Record[OpNum++]) {
+          unsigned PtrProvenanceTypeID;
+          if (getValueTypePair(Record, OpNum, NextValueNo, PtrProvenance,
+                               PtrProvenanceTypeID, CurBB))
+            return error("Invalid ptr_provenance operand");
+        }
+      }
+      if (OpNum != Record.size())
+        return error("Invalid record");
+      I = new LoadInst(Ty, Op, "", IsVolatile, *Align, Ordering, SSID);
+      if (PtrProvenance)
+        cast<LoadInst>(I)->setPtrProvenanceOperand(PtrProvenance);
       InstructionList.push_back(I);
       break;
     }
     case bitc::FUNC_CODE_INST_STORE:
-    case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, vol]
+    case bitc::FUNC_CODE_INST_STORE_OLD: {
+      // STORE2:[ptrty, ptr, val, align, vol, [hasPtrProv, [PtrProv]]]
       unsigned OpNum = 0;
       Value *Val, *Ptr;
       unsigned PtrTypeID, ValTypeID;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, PtrTypeID, CurBB))
         return error("Invalid record");
 
       if (BitCode == bitc::FUNC_CODE_INST_STORE) {
         if (getValueTypePair(Record, OpNum, NextValueNo, Val, ValTypeID, CurBB))
           return error("Invalid record");
       } else {
         ValTypeID = getContainedTypeID(PtrTypeID);
         if (popValue(Record, OpNum, NextValueNo, getTypeByID(ValTypeID),
                      ValTypeID, Val, CurBB))
           return error("Invalid record");
       }
 
-      if (OpNum + 2 != Record.size())
+      if (OpNum + 2 > Record.size())
         return error("Invalid record");
 
       if (Error Err = typeCheckLoadStoreInst(Val->getType(), Ptr->getType()))
         return Err;
       MaybeAlign Align;
       if (Error Err = parseAlignmentValue(Record[OpNum], Align))
         return Err;
       SmallPtrSet<Type *, 4> Visited;
       if (!Align && !Val->getType()->isSized(&Visited))
         return error("store of unsized type");
       if (!Align)
         Align = TheModule->getDataLayout().getABITypeAlign(Val->getType());
-      I = new StoreInst(Val, Ptr, Record[OpNum + 1], *Align);
+      bool IsVolatile = Record[OpNum + 1];
+      OpNum += 2;
+      Value *PtrProvenance = nullptr;
+      if (OpNum < Record.size()) {
+        // Do we have a ptr_provenance ?
+        if (Record[OpNum++]) {
+          unsigned PtrProvenanceTypeID;
+          if (getValueTypePair(Record, OpNum, NextValueNo, PtrProvenance,
+                               PtrProvenanceTypeID, CurBB))
+            return error("Invalid ptr_provenance operand");
+        }
+      }
+      if (OpNum != Record.size())
+        return error("Invalid record");
+      I = new StoreInst(Val, Ptr, IsVolatile, *Align);
+      if (PtrProvenance)
+        cast<StoreInst>(I)->setPtrProvenanceOperand(PtrProvenance);
       InstructionList.push_back(I);
       break;
     }
     case bitc::FUNC_CODE_INST_STOREATOMIC:
     case bitc::FUNC_CODE_INST_STOREATOMIC_OLD: {
-      // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, ssid]
+      // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, ssid,
+      //               [hasPtrProv, [PtrProv]]]
       unsigned OpNum = 0;
       Value *Val, *Ptr;
       unsigned PtrTypeID, ValTypeID;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, PtrTypeID, CurBB) ||
           !isa<PointerType>(Ptr->getType()))
         return error("Invalid record");
       if (BitCode == bitc::FUNC_CODE_INST_STOREATOMIC) {
         if (getValueTypePair(Record, OpNum, NextValueNo, Val, ValTypeID, CurBB))
           return error("Invalid record");
       } else {
         ValTypeID = getContainedTypeID(PtrTypeID);
         if (popValue(Record, OpNum, NextValueNo, getTypeByID(ValTypeID),
                      ValTypeID, Val, CurBB))
           return error("Invalid record");
       }
 
-      if (OpNum + 4 != Record.size())
+      if (OpNum + 4 > Record.size())
         return error("Invalid record");
 
       if (Error Err = typeCheckLoadStoreInst(Val->getType(), Ptr->getType()))
         return Err;
       AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]);
       if (Ordering == AtomicOrdering::NotAtomic ||
           Ordering == AtomicOrdering::Acquire ||
           Ordering == AtomicOrdering::AcquireRelease)
         return error("Invalid record");
       SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]);
       if (Ordering != AtomicOrdering::NotAtomic && Record[OpNum] == 0)
         return error("Invalid record");
 
       MaybeAlign Align;
       if (Error Err = parseAlignmentValue(Record[OpNum], Align))
         return Err;
       if (!Align)
         return error("Alignment missing from atomic store");
-      I = new StoreInst(Val, Ptr, Record[OpNum + 1], *Align, Ordering, SSID);
+      bool IsVolatile = Record[OpNum + 1];
+      OpNum += 4;
+      Value *PtrProvenance = nullptr;
+      if (OpNum < Record.size()) {
+        // Do we have a ptr_provenance ?
+        if (Record[OpNum++]) {
+          unsigned PtrProvenanceTypeID;
+          if (getValueTypePair(Record, OpNum, NextValueNo, PtrProvenance,
+                               PtrProvenanceTypeID, CurBB))
+            return error("Invalid ptr_provenance operand");
+        }
+      }
+      if (OpNum != Record.size())
+        return error("Invalid record");
+      I = new StoreInst(Val, Ptr, IsVolatile, *Align, Ordering, SSID);
+      if (PtrProvenance)
+        cast<StoreInst>(I)->setPtrProvenanceOperand(PtrProvenance);
       InstructionList.push_back(I);
       break;
     }
     case bitc::FUNC_CODE_INST_CMPXCHG_OLD: {
       // CMPXCHG_OLD: [ptrty, ptr, cmp, val, vol, ordering, synchscope,
       // failure_ordering?, weak?]
       const size_t NumRecords = Record.size();
       unsigned OpNum = 0;

llvm/lib/Bitcode/Writer/BitcodeWriter.cpp

if (C->isNullValue()) {
           isCStrChar6 = BitCodeAbbrevOp::isChar6(V);
       }
 
       if (isCStrChar6)
         AbbrevToUse = CString6Abbrev;
       else if (isCStr7)
         AbbrevToUse = CString7Abbrev;
     } else if (const ConstantDataSequential *CDS =
                    dyn_cast<ConstantDataSequential>(C)) {
       Code = bitc::CST_CODE_DATA;
       Type *EltTy = CDS->getElementType();
       if (isa<IntegerType>(EltTy)) {
         for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
           Record.push_back(CDS->getElementAsInteger(i));
       } else {
         for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
           Record.push_back(
void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
   }
 
   case Instruction::Load:
     if (cast<LoadInst>(I).isAtomic()) {
       Code = bitc::FUNC_CODE_INST_LOADATOMIC;
       pushValueAndType(I.getOperand(0), InstID, Vals);
     } else {
       Code = bitc::FUNC_CODE_INST_LOAD;
-      if (!pushValueAndType(I.getOperand(0), InstID, Vals)) // ptr
+      if (!pushValueAndType(I.getOperand(0), InstID, Vals) &&
+          !cast<LoadInst>(I).hasPtrProvenanceOperand()) // ptr
         AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
     }
     Vals.push_back(VE.getTypeID(I.getType()));
     Vals.push_back(getEncodedAlign(cast<LoadInst>(I).getAlign()));
     Vals.push_back(cast<LoadInst>(I).isVolatile());
     if (cast<LoadInst>(I).isAtomic()) {
       Vals.push_back(getEncodedOrdering(cast<LoadInst>(I).getOrdering()));
       Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(I).getSyncScopeID()));
     }
+    if (cast<LoadInst>(I).hasPtrProvenanceOperand()) {
+      Vals.push_back(true); // ptr_provenance present
+      pushValueAndType(cast<LoadInst>(I).getPtrProvenanceOperand(), InstID,
+                       Vals); // ptrty + ptr_provenance
+    } else {
+      // No ptr_provenance - do nothing for now, when in future more arguments
+      // are emitted, do not forget to emit a 'false':
+      // Vals.push_back(false);
+    }
     break;
   case Instruction::Store:
     if (cast<StoreInst>(I).isAtomic())
       Code = bitc::FUNC_CODE_INST_STOREATOMIC;
     else
       Code = bitc::FUNC_CODE_INST_STORE;
     pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr
     pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val
     Vals.push_back(getEncodedAlign(cast<StoreInst>(I).getAlign()));
     Vals.push_back(cast<StoreInst>(I).isVolatile());
     if (cast<StoreInst>(I).isAtomic()) {
       Vals.push_back(getEncodedOrdering(cast<StoreInst>(I).getOrdering()));
       Vals.push_back(
           getEncodedSyncScopeID(cast<StoreInst>(I).getSyncScopeID()));
     }
+    if (cast<StoreInst>(I).hasPtrProvenanceOperand()) {
+      Vals.push_back(true); // ptr_provenance present
+      pushValueAndType(cast<StoreInst>(I).getPtrProvenanceOperand(), InstID,
+                       Vals); // ptrty + ptr_provenance
+    } else {
+      // No ptr_provenance - do nothing for now, when in future more arguments
+      // are emitted, do not forget to emit a 'false':
+      // Vals.push_back(false);
+    }
     break;
   case Instruction::AtomicCmpXchg:
     Code = bitc::FUNC_CODE_INST_CMPXCHG;
     pushValueAndType(I.getOperand(0), InstID, Vals); // ptrty + ptr
     pushValueAndType(I.getOperand(1), InstID, Vals); // cmp.
     pushValue(I.getOperand(2), InstID, Vals);        // newval.
     Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile());
     Vals.push_back(

llvm/test/Bitcode/compatibility.ll

define void @atomics(ptr %word) {
 
   %ld.1 = load atomic i32, ptr %word monotonic, align 4
   ; CHECK: %ld.1 = load atomic i32, ptr %word monotonic, align 4
   %ld.2 = load atomic volatile i32, ptr %word acquire, align 8
   ; CHECK: %ld.2 = load atomic volatile i32, ptr %word acquire, align 8
   %ld.3 = load atomic volatile i32, ptr %word syncscope("singlethread") seq_cst, align 16
   ; CHECK: %ld.3 = load atomic volatile i32, ptr %word syncscope("singlethread") seq_cst, align 16
 
+  %ld.1p = load atomic i32, ptr %word monotonic, ptr_provenance ptr %word, align 4
+  ; CHECK: %ld.1p = load atomic i32, ptr %word monotonic, ptr_provenance ptr %word, align 4
+  %ld.2p = load atomic volatile i32, ptr %word acquire, ptr_provenance ptr %word, align 8
+  ; CHECK: %ld.2p = load atomic volatile i32, ptr %word acquire, ptr_provenance ptr %word, align 8
+  %ld.3p = load atomic volatile i32, ptr %word syncscope("singlethread") seq_cst, ptr_provenance ptr %word, align 16
+  ; CHECK: %ld.3p = load atomic volatile i32, ptr %word syncscope("singlethread") seq_cst, ptr_provenance ptr %word, align 16
+
   store atomic i32 23, ptr %word monotonic, align 4
   ; CHECK: store atomic i32 23, ptr %word monotonic, align 4
   store atomic volatile i32 24, ptr %word monotonic, align 4
   ; CHECK: store atomic volatile i32 24, ptr %word monotonic, align 4
   store atomic volatile i32 25, ptr %word syncscope("singlethread") monotonic, align 4
   ; CHECK: store atomic volatile i32 25, ptr %word syncscope("singlethread") monotonic, align 4
+
+  store atomic i32 26, ptr %word monotonic, ptr_provenance ptr %word, align 4
+  ; CHECK: store atomic i32 26, ptr %word monotonic, ptr_provenance ptr %word, align 4
+  store atomic volatile i32 27, ptr %word monotonic, ptr_provenance ptr %word, align 4
+  ; CHECK: store atomic volatile i32 27, ptr %word monotonic, ptr_provenance ptr %word, align 4
+  store atomic volatile i32 28, ptr %word syncscope("singlethread") monotonic, ptr_provenance ptr %word, align 4
+  ; CHECK: store atomic volatile i32 28, ptr %word syncscope("singlethread") monotonic, ptr_provenance ptr %word, align 4
   ret void
 }
 
 define void @fp_atomics(ptr %word) {
 ; CHECK: %atomicrmw.xchg = atomicrmw xchg ptr %word, float 1.000000e+00 monotonic
   %atomicrmw.xchg = atomicrmw xchg ptr %word, float 1.0 monotonic
 
 ; CHECK: %atomicrmw.fadd = atomicrmw fadd ptr %word, float 1.000000e+00 monotonic
define void @instructions.memops(ptr %base) {
   alloca inalloca i32, i8 4, align 4
   ; CHECK: alloca inalloca i32, i8 4, align 4
 
   load ptr, ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
   ; CHECK: load ptr, ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
   load volatile ptr, ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
   ; CHECK: load volatile ptr, ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
 
+  load ptr, ptr %base, ptr_provenance ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
+  ; CHECK: load ptr, ptr %base, ptr_provenance ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
+  load volatile ptr, ptr %base, ptr_provenance ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
+  ; CHECK: load volatile ptr, ptr %base, ptr_provenance ptr %base, align 8, !invariant.load !7, !nontemporal !8, !nonnull !8, !dereferenceable !9, !dereferenceable_or_null !9
+
   store ptr null, ptr %base, align 4, !nontemporal !8
   ; CHECK: store ptr null, ptr %base, align 4, !nontemporal !8
   store volatile ptr null, ptr %base, align 4, !nontemporal !8
   ; CHECK: store volatile ptr null, ptr %base, align 4, !nontemporal !8
 
+  store ptr null, ptr %base, ptr_provenance ptr %base, align 4, !nontemporal !8
+  ; CHECK: store ptr null, ptr %base, ptr_provenance ptr %base, align 4, !nontemporal !8
+  store volatile ptr null, ptr %base, ptr_provenance ptr %base, align 4, !nontemporal !8
+  ; CHECK: store volatile ptr null, ptr %base, ptr_provenance ptr %base, align 4, !nontemporal !8
+
   ret void
 }
 
 ; Instructions -- Conversion Operations
 define void @instructions.conversions() {
   trunc i32 -1 to i1
   ; CHECK: trunc i32 -1 to i1
   zext i32 -1 to i64

llvm/test/Bitcode/loadstore_ptr_provenance.ll

 ; RUN: opt -passes=verify -S < %s | FileCheck %s
-; Activate when bitcode support is added:
-; R U N: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
-; R U N: verify-uselistorder < %s
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+; RUN: verify-uselistorder < %s
 
 define i32 @f(ptr %p, ptr %q, ptr %word, ptr %base) {
   ; CHECK:      define i32 @f(ptr %p, ptr %q, ptr %word, ptr %base) {
   store i32 42, ptr %p, ptr_provenance ptr %p
   ; CHECK-NEXT:   store i32 42, ptr %p, ptr_provenance ptr %p
   store i32 43, ptr %q, ptr_provenance ptr %q
   ; CHECK-NEXT:   store i32 43, ptr %q, ptr_provenance ptr %q
   %r = load i32, ptr %p, ptr_provenance ptr %p