Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -2582,6 +2582,14 @@ ``"ptrauth"`` operand bundle tag. They are described in the `Pointer Authentication `__ document. +Coroutine Readnone Operand Bundles +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A ``"coro_readnone"`` operand bundle on a call indicates the call is a readnone +call in a coroutine. A call with ``"coro_readnone"`` operand bundle is only allowed +to appear in a coroutine. The ``"coro_readnone"`` operand bundles would be eliminated +after the coroutine get splitted. + .. _moduleasm: Module-Level Inline Assembly Index: llvm/include/llvm/IR/LLVMContext.h =================================================================== --- llvm/include/llvm/IR/LLVMContext.h +++ llvm/include/llvm/IR/LLVMContext.h @@ -95,6 +95,7 @@ OB_gc_live = 5, // "gc-live" OB_clang_arc_attachedcall = 6, // "clang.arc.attachedcall" OB_ptrauth = 7, // "ptrauth" + OB_coro_readnone = 8, // "coro_readnone" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. Index: llvm/lib/IR/LLVMContext.cpp =================================================================== --- llvm/lib/IR/LLVMContext.cpp +++ llvm/lib/IR/LLVMContext.cpp @@ -87,6 +87,11 @@ "ptrauth operand bundle id drifted!"); (void)PtrauthEntry; + auto *CoroReadnoneEntry = pImpl->getOrInsertBundleTag("coro_readnone"); + assert(CoroReadnoneEntry->second == LLVMContext::OB_coro_readnone && + "coro_readnone operand bundle id drifted!"); + (void)CoroReadnoneEntry; + SyncScope::ID SingleThreadSSID = pImpl->getOrInsertSyncScopeID("singlethread"); assert(SingleThreadSSID == SyncScope::SingleThread && Index: llvm/lib/Transforms/Coroutines/CoroCleanup.cpp =================================================================== --- llvm/lib/Transforms/Coroutines/CoroCleanup.cpp +++ llvm/lib/Transforms/Coroutines/CoroCleanup.cpp @@ -50,7 +50,26 @@ bool Changed = false; for (Instruction &I : llvm::make_early_inc_range(instructions(F))) { - if (auto *II = dyn_cast(&I)) { + auto *CB = dyn_cast(&I); + if (!CB) + continue; + + // Convert "coro_readnone" operand bundles to normal ReadNone attribute + // to enable further optimizations. + if (CB->getOperandBundle("coro_readnone")) { + auto *NewCB = + CallBase::removeOperandBundle(CB, LLVMContext::OB_coro_readnone, CB); + NewCB->copyMetadata(*CB); + NewCB->addFnAttr(Attribute::ReadNone); + NewCB->removeFnAttr(Attribute::ReadOnly); + NewCB->removeFnAttr(Attribute::InaccessibleMemOnly); + + CB->replaceAllUsesWith(NewCB); + CB->eraseFromParent(); + CB = NewCB; + } + + if (auto *II = dyn_cast(CB)) { switch (II->getIntrinsicID()) { default: continue; Index: llvm/lib/Transforms/Coroutines/CoroEarly.cpp =================================================================== --- llvm/lib/Transforms/Coroutines/CoroEarly.cpp +++ llvm/lib/Transforms/Coroutines/CoroEarly.cpp @@ -149,11 +149,57 @@ CoroIdInst *CoroId = nullptr; SmallVector CoroFrees; bool HasCoroSuspend = false; + for (Instruction &I : llvm::make_early_inc_range(instructions(F))) { auto *CB = dyn_cast(&I); if (!CB) continue; + // This is a workaround for the bug about pthread_self() in coroutine. + // See https://github.com/llvm/llvm-project/issues/47177 for the + // background. The reason behind the bug is that pthread_self() is + // marked as + // __attribute__((__const__)) which would be converted to `readnone` + // when get lowered to LLVM IR. The readnone function without parameters + // implies that all the calls to the function would return the same + // result. So the compiler would optimize the following code: + // + // auto a = pthread_self(); + // co_await something(); + // auto b = pthread_self(); + // + // to + // + // auto a = pthread_self(); + // co_await something(); + // // replace uses of b with a + // + // The transformation is incorrect in case the coroutine might resume in + // another thread. The key reason for the bug is the abuse of + // __attribute__((__const__)) for thread identification. However, it is + // not easy to fix the problem in the library side (we don't know if + // there are other similar problems in other places) and ask the end + // user to update the library. So we choose to block the optimization + // before we split coroutine by replacing readnone attribute as a + // placeholder so that we could rewrite readnone after we lowered + // coroutines. + if (F.isPresplitCoroutine() && CB->hasFnAttr(Attribute::ReadNone) && + !isa(CB)) { + OperandBundleDef OB("coro_readnone", std::vector{}); + auto *NewCall = + CallBase::addOperandBundle(CB, LLVMContext::OB_coro_readnone, OB, CB); + NewCall->copyMetadata(*CB); + NewCall->removeFnAttr(Attribute::ReadNone); + // We add readonly and inaccessiblememonly attributes here to enable + // optimizations as much as possible. + NewCall->addFnAttr(Attribute::ReadOnly); + NewCall->addFnAttr(Attribute::InaccessibleMemOnly); + + CB->replaceAllUsesWith(NewCall); + CB->eraseFromParent(); + CB = NewCall; + } + switch (CB->getIntrinsicID()) { default: continue; Index: llvm/lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- llvm/lib/Transforms/Utils/InlineFunction.cpp +++ llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1777,6 +1777,10 @@ if (Tag == LLVMContext::OB_clang_arc_attachedcall) continue; + // We could inline calls with "coro_readnone" directly. + if (Tag == LLVMContext::OB_coro_readnone) + continue; + return InlineResult::failure("unsupported operand bundle"); } } @@ -2189,6 +2193,28 @@ // 'nounwind'. if (MarkNoUnwind) CI->setDoesNotThrow(); + + // Due to legacy reasons, the readnone function might read the thread + // identification. It is fine before we introduced coroutines since a + // coroutine might resume in another thread. Here we remove the readnone + // attibute when we inline readnone calls to an unlowered coroutines to + // disable the incorrect optimizations. The readnone attribute would be + // added back after the coroutine get lowered. + if (Caller->isPresplitCoroutine() && + CI->hasFnAttr(Attribute::ReadNone) && !isa(CI)) { + OperandBundleDef OB("coro_readnone", std::vector{}); + auto *NewCI = CallBase::addOperandBundle( + CI, LLVMContext::OB_coro_readnone, OB, CI); + NewCI->copyMetadata(*CI); + NewCI->removeFnAttr(Attribute::ReadNone); + // We add readonly and inaccessiblememonly attributes here to enable + // optimizations as much as possible. + NewCI->addFnAttr(Attribute::ReadOnly); + NewCI->addFnAttr(Attribute::InaccessibleMemOnly); + + CI->replaceAllUsesWith(NewCI); + CI->eraseFromParent(); + } } } } Index: llvm/test/Bitcode/operand-bundles-bc-analyzer.ll =================================================================== --- llvm/test/Bitcode/operand-bundles-bc-analyzer.ll +++ llvm/test/Bitcode/operand-bundles-bc-analyzer.ll @@ -11,6 +11,7 @@ ; CHECK-NEXT: