diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -1561,6 +1561,16 @@ static CallInst *Create(CallInst *CI, ArrayRef Bundles, Instruction *InsertPt = nullptr); + /// Create a clone of \p CI with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned call instruction is identical \p CI in every way except that + /// the operand bundle for the new instruction is set to the operand bundle + /// in \p Bundle. + static CallInst *CreateWithReplacedBundle(CallInst *CI, + OperandBundleDef Bundle, + Instruction *InsertPt = nullptr); + /// Generate the IR for a call to malloc: /// 1. Compute the malloc call's argument as the specified type's size, /// possibly multiplied by the array size if the array size is not @@ -3778,6 +3788,16 @@ static InvokeInst *Create(InvokeInst *II, ArrayRef Bundles, Instruction *InsertPt = nullptr); + /// Create a clone of \p II with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned invoke instruction is identical to \p II in every way except + /// that the operand bundle for the new instruction is set to the operand + /// bundle in \p Bundle. + static InvokeInst *CreateWithReplacedBundle(InvokeInst *II, + OperandBundleDef Bundles, + Instruction *InsertPt = nullptr); + // get*Dest - Return the destination basic blocks... BasicBlock *getNormalDest() const { return cast(Op()); diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -515,6 +515,18 @@ return NewCI; } +CallInst *CallInst::CreateWithReplacedBundle(CallInst *CI, OperandBundleDef OpB, + Instruction *InsertPt) { + SmallVector OpDefs; + for (unsigned i = 0, e = CI->getNumOperandBundles(); i < e; ++i) { + auto ChildOB = CI->getOperandBundleAt(i); + if (ChildOB.getTagName() != OpB.getTag()) + OpDefs.emplace_back(ChildOB); + } + OpDefs.emplace_back(OpB); + return CallInst::Create(CI, OpDefs, InsertPt); +} + // Update profile weight for call instruction by scaling it using the ratio // of S/T. The meaning of "branch_weights" meta data for call instruction is // transfered to represent call count. @@ -826,6 +838,18 @@ return NewII; } +InvokeInst *InvokeInst::CreateWithReplacedBundle(InvokeInst *II, + OperandBundleDef OpB, + Instruction *InsertPt) { + SmallVector OpDefs; + for (unsigned i = 0, e = II->getNumOperandBundles(); i < e; ++i) { + auto ChildOB = II->getOperandBundleAt(i); + if (ChildOB.getTagName() != OpB.getTag()) + OpDefs.emplace_back(ChildOB); + } + OpDefs.emplace_back(OpB); + return InvokeInst::Create(II, OpDefs, InsertPt); +} LandingPadInst *InvokeInst::getLandingPadInst() const { return cast(getUnwindDest()->getFirstNonPHI()); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -1487,7 +1487,8 @@ break; } case Intrinsic::experimental_gc_statepoint: { - auto &GCSP = *cast(II); + GCStatepointInst &GCSP = *cast(II); + SmallPtrSet LiveGcValues; for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) { GCRelocateInst &GCR = *const_cast(Reloc); @@ -1539,7 +1540,48 @@ // Canonicalize on the type from the uses to the defs // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...) + LiveGcValues.insert(BasePtr); + LiveGcValues.insert(DerivedPtr); } + Optional Bundle = + GCSP.getOperandBundle(LLVMContext::OB_gc_live); + unsigned NumOfGCLives = LiveGcValues.size(); + if (!Bundle.hasValue() || NumOfGCLives == Bundle->Inputs.size()) + break; + // We can reduce the size of gc live bundle. + DenseMap Val2Idx; + std::vector NewLiveGc; + for (unsigned I = 0, E = Bundle->Inputs.size(); I < E; ++I) { + Value *V = Bundle->Inputs[I]; + if (Val2Idx.count(V)) + continue; + if (LiveGcValues.count(V)) { + Val2Idx[V] = NewLiveGc.size(); + NewLiveGc.push_back(V); + } else + Val2Idx[V] = NumOfGCLives; + } + // Update all gc.relocates + for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) { + GCRelocateInst &GCR = *const_cast(Reloc); + Value *BasePtr = GCR.getBasePtr(); + assert(Val2Idx.count(BasePtr) && Val2Idx[BasePtr] != NumOfGCLives && + "Missed live gc for base pointer"); + auto *OpIntTy1 = GCR.getOperand(1)->getType(); + GCR.setOperand(1, ConstantInt::get(OpIntTy1, Val2Idx[BasePtr])); + Value *DerivedPtr = GCR.getDerivedPtr(); + assert(Val2Idx.count(DerivedPtr) && Val2Idx[DerivedPtr] != NumOfGCLives && + "Missed live gc for derived pointer"); + auto *OpIntTy2 = GCR.getOperand(2)->getType(); + GCR.setOperand(2, ConstantInt::get(OpIntTy2, Val2Idx[DerivedPtr])); + } + // Create new statepoint instruction. + OperandBundleDef NewBundle("gc-live", NewLiveGc); + if (isa(II)) + return CallInst::CreateWithReplacedBundle(cast(II), NewBundle); + else + return InvokeInst::CreateWithReplacedBundle(cast(II), + NewBundle); break; } case Intrinsic::experimental_guard: { diff --git a/llvm/test/Transforms/InstCombine/statepoint-iter.ll b/llvm/test/Transforms/InstCombine/statepoint-iter.ll --- a/llvm/test/Transforms/InstCombine/statepoint-iter.ll +++ b/llvm/test/Transforms/InstCombine/statepoint-iter.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -instcombine -instcombine-max-iterations=1 -S | FileCheck %s ; These tests check the optimizations specific to ; pointers being relocated at a statepoint. @@ -6,6 +7,18 @@ declare void @func() define i1 @test_null(i1 %cond) gc "statepoint-example" { +; CHECK-LABEL: @test_null( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]] +; CHECK: right: +; CHECK-NEXT: br label [[MERGE:%.*]] +; CHECK: left: +; CHECK-NEXT: [[SAFEPOINT_TOKEN:%.*]] = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* nonnull @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"() ] +; CHECK-NEXT: br label [[MERGE]] +; CHECK: merge: +; CHECK-NEXT: [[SAFEPOINT_TOKEN2:%.*]] = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* nonnull @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"() ] +; CHECK-NEXT: ret i1 true +; entry: br i1 %cond, label %left, label %right @@ -23,13 +36,21 @@ %pnew2 = call i32* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 0) %cmp = icmp eq i32* %pnew2, null ret i1 %cmp -; CHECK-LABEL: test_null -; CHECK-NOT: %pnew -; CHECK-NOT: %pnew2 -; CHECK: ret i1 true } define i32* @test_undef(i1 %cond) gc "statepoint-example" { +; CHECK-LABEL: @test_undef( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND:%.*]], label [[LEFT:%.*]], label [[RIGHT:%.*]] +; CHECK: right: +; CHECK-NEXT: br label [[MERGE:%.*]] +; CHECK: left: +; CHECK-NEXT: [[SAFEPOINT_TOKEN:%.*]] = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* nonnull @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"() ] +; CHECK-NEXT: br label [[MERGE]] +; CHECK: merge: +; CHECK-NEXT: [[SAFEPOINT_TOKEN2:%.*]] = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* nonnull @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"() ] +; CHECK-NEXT: ret i32* undef +; entry: br i1 %cond, label %left, label %right @@ -46,10 +67,6 @@ %safepoint_token2 = tail call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live" (i32* %pnew_phi)] %pnew2 = call i32* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token2, i32 0, i32 0) ret i32* %pnew2 -; CHECK-LABEL: test_undef -; CHECK-NOT: %pnew -; CHECK-NOT: %pnew2 -; CHECK: ret i32* undef } declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)