diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -45,3 +45,5 @@ LLVM_FIXED_MD_KIND(MD_nosanitize, "nosanitize", 31) LLVM_FIXED_MD_KIND(MD_func_sanitize, "func_sanitize", 32) LLVM_FIXED_MD_KIND(MD_exclude, "exclude", 33) +LLVM_FIXED_MD_KIND(MD_memprof, "memprof", 34) +LLVM_FIXED_MD_KIND(MD_callsite, "callsite", 35) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -469,6 +469,9 @@ void visitRangeMetadata(Instruction &I, MDNode *Range, Type *Ty); void visitDereferenceableMetadata(Instruction &I, MDNode *MD); void visitProfMetadata(Instruction &I, MDNode *MD); + void visitCallStackMetadata(MDNode *MD); + void visitMemProfMetadata(Instruction &I, MDNode *MD); + void visitCallsiteMetadata(Instruction &I, MDNode *MD); void visitAnnotationMetadata(MDNode *Annotation); void visitAliasScopeMetadata(const MDNode *MD); void visitAliasScopeListMetadata(const MDNode *MD); @@ -4487,6 +4490,55 @@ } } +void Verifier::visitCallStackMetadata(MDNode *MD) { + // Call stack metadata should consist of a list of at least 1 constant int + // (representing a hash of the location). + Check(MD->getNumOperands() >= 1, + "call stack metadata should have at least 1 operand", MD); + + for (const auto &Op : MD->operands()) + Check(mdconst::dyn_extract_or_null(Op), + "call stack metadata operand should be constant integer", Op); +} + +void Verifier::visitMemProfMetadata(Instruction &I, MDNode *MD) { + Check(isa(I), "!memprof metadata should only exist on calls", &I); + Check(MD->getNumOperands() >= 1, + "!memprof annotations should have at least 1 metadata operand " + "(MemInfoBlock)", + MD); + + // Check each MIB + for (auto &MIBOp : MD->operands()) { + MDNode *MIB = dyn_cast(MIBOp); + // The first operand of an MIB should be the call stack metadata. + // There rest of the operands should be MDString tags, and there should be + // at least one. + Check(MIB->getNumOperands() >= 2, + "Each !memprof MemInfoBlock should have at least 2 operands", MIB); + + // Check call stack metadata (first operand). + Check(MIB->getOperand(0) != nullptr, + "!memprof MemInfoBlock first operand should not be null", MIB); + Check(isa(MIB->getOperand(0)), + "!memprof MemInfoBlock first operand should be an MDNode", MIB); + MDNode *StackMD = dyn_cast(MIB->getOperand(0)); + visitCallStackMetadata(StackMD); + + // Check that remaining operands are MDString. + Check(std::all_of(MIB->op_begin() + 1, MIB->op_end(), + [](const MDOperand &Op) { return isa(Op); }), + "Not all !memprof MemInfoBlock operands 1 to N are MDString", MIB); + } +} + +void Verifier::visitCallsiteMetadata(Instruction &I, MDNode *MD) { + Check(isa(I), "!callsite metadata should only exist on calls", &I); + // Verify the partial callstack annotated from memprof profiles. This callsite + // is a part of a profiled allocation callstack. + visitCallStackMetadata(MD); +} + void Verifier::visitAnnotationMetadata(MDNode *Annotation) { Check(isa(Annotation), "annotation must be a tuple"); Check(Annotation->getNumOperands() >= 1, @@ -4733,6 +4785,12 @@ if (MDNode *MD = I.getMetadata(LLVMContext::MD_prof)) visitProfMetadata(I, MD); + if (MDNode *MD = I.getMetadata(LLVMContext::MD_memprof)) + visitMemProfMetadata(I, MD); + + if (MDNode *MD = I.getMetadata(LLVMContext::MD_callsite)) + visitCallsiteMetadata(I, MD); + if (MDNode *Annotation = I.getMetadata(LLVMContext::MD_annotation)) visitAnnotationMetadata(Annotation); diff --git a/llvm/test/Verifier/memprof-metadata-bad.ll b/llvm/test/Verifier/memprof-metadata-bad.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Verifier/memprof-metadata-bad.ll @@ -0,0 +1,56 @@ +; Test that incorrect memprof and callsite metadata fail verification. +; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s + +define i32* @test1() { +entry: + %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0 + %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !1 + %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !3 + %call4 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !5 + %call5 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !7, !callsite !9 + %0 = bitcast i8* %call5 to i32* + ret i32* %0 +} + +define i32* @test2() { +entry: + %call = call noundef i32* @test1(), !callsite !10 + ret i32* %call +} + +define i32* @test3() { +entry: + %call = call noundef i32* @test2(), !callsite !11 + ret i32* %call +} + +define void @wronginsttype() { + %1 = add i32 0, 1, !memprof !0, !callsite !9 + ret void +} + +declare dso_local noalias noundef i8* @malloc(i64 noundef) + +; CHECK: !memprof annotations should have at least 1 metadata operand (MemInfoBlock) +!0 = !{} +!1 = !{!2} +; CHECK: !memprof MemInfoBlock first operand should not be null +!2 = !{null, !"cold"} +!3 = !{!4} +; CHECK: !memprof MemInfoBlock first operand should be an MDNode +!4 = !{i64 0, !"cold"} +!5 = !{!6} +; CHECK: Each !memprof MemInfoBlock should have at least 2 operands +!6 = !{i64 0} +!7 = !{!8} +; CHECK: call stack metadata should have at least 1 operand +; CHECK: Not all !memprof MemInfoBlock operands 1 to N are MDString +!8 = !{!0, !"default", i64 0} +!9 = !{i64 123} +; CHECK: call stack metadata operand should be constant integer +!10 = !{!"wrongtype"} +!11 = !{i64 789, i64 678} + +; Errors from annotating incorrect instruction type in @wronginsttype. +; CHECK: !memprof metadata should only exist on calls +; CHECK: !callsite metadata should only exist on calls diff --git a/llvm/test/Verifier/memprof-metadata-good.ll b/llvm/test/Verifier/memprof-metadata-good.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Verifier/memprof-metadata-good.ll @@ -0,0 +1,34 @@ +; Test that well-formed memprof and callsite metadata pass verification. +; RUN: llvm-as -disable-output < %s 2>&1 + +define i32* @test1() { +entry: + %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0, !callsite !5 + %0 = bitcast i8* %call to i32* + ret i32* %0 +} + +define i32* @test2() { +entry: + %call = call noundef i32* @test1(), !callsite !6 + ret i32* %call +} + +define i32* @test3() { +entry: + %call = call noundef i32* @test1(), !callsite !7 + ret i32* %call +} + +declare dso_local noalias noundef i8* @malloc(i64 noundef) + +!0 = !{!1, !3} +; !memprof metadata should be able to support an arbitrary list of string tags. +!1 = !{!2, !"default", !"tag2"} +!2 = !{i64 123, i64 456} +!3 = !{!4, !"cold", !"tag3", !"tag4"} +!4 = !{i64 123, i64 789, i64 678} +!5 = !{i64 123} +!6 = !{i64 456} +; Inlined callsites will have more than one call stack id. +!7 = !{i64 789, i64 678}