diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h --- a/clang/include/clang/Sema/Scope.h +++ b/clang/include/clang/Sema/Scope.h @@ -199,6 +199,14 @@ using DeclSetTy = llvm::SmallPtrSet; DeclSetTy DeclsInScope; + /// NRVOCandidatesInScope - This keeps track of all unspoiled NRVO candidates + /// in this scope. When a VarDecl is added to the scope, it is added to + /// NRVOCandidatesInScope. When `addNRVOCandidate` is called, all variables + /// except the actual candidate are removed from the set. The set is used + /// to calculate the `NRVO` variable. + using NRVOCandidatesSetTy = llvm::SmallPtrSet; + NRVOCandidatesSetTy NRVOCandidatesInScope; + /// The DeclContext with which this scope is associated. For /// example, the entity of a class scope is the class itself, the /// entity of a function scope is a function, etc. @@ -210,7 +218,7 @@ /// Used to determine if errors occurred in this scope. DiagnosticErrorTrap ErrorTrap; - /// A lattice consisting of undefined, a single NRVO candidate variable in + /// A lattice consisting of undefined, the single NRVO candidate variable in /// this scope, or over-defined. The bit is true when over-defined. llvm::PointerIntPair NRVO; @@ -305,6 +313,8 @@ void AddDecl(Decl *D) { DeclsInScope.insert(D); + if (VarDecl *VD = dyn_cast(D)) + NRVOCandidatesInScope.insert(VD); } void RemoveDecl(Decl *D) { @@ -506,18 +516,28 @@ } void addNRVOCandidate(VarDecl *VD) { - if (NRVO.getInt()) - return; - if (NRVO.getPointer() == nullptr) { - NRVO.setPointer(VD); - return; - } - if (NRVO.getPointer() != VD) - setNoNRVO(); + // every candidate except VD is "spoiled" now, remove them from the set + bool HasCandidate = NRVOCandidatesInScope.contains(VD); + NRVOCandidatesInScope.clear(); + if (HasCandidate) + NRVOCandidatesInScope.insert(VD); + + // the variable may have NRVO if it's an existing candidate or an outer + // variable + VarDecl *NewNRVO = nullptr; + if (HasCandidate || !isDeclScope(VD)) + NewNRVO = VD; + + // if we changed the candidate, the NRVO for the parent scope is + // over-defined + if (NRVO.getPointer() != nullptr && NRVO.getPointer() != NewNRVO) + NRVO.setInt(true); + NRVO.setPointer(NewNRVO); } void setNoNRVO() { NRVO.setInt(true); + NRVOCandidatesInScope.clear(); NRVO.setPointer(nullptr); } diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp --- a/clang/lib/Sema/Scope.cpp +++ b/clang/lib/Sema/Scope.cpp @@ -194,7 +194,7 @@ OS << "Entity : (clang::DeclContext*)" << DC << '\n'; if (NRVO.getInt()) - OS << "NRVO not allowed\n"; - else if (NRVO.getPointer()) + OS << "Invalidates NRVO candidates in parent\n"; + if (NRVO.getPointer()) OS << "NRVO candidate : (clang::VarDecl*)" << NRVO.getPointer() << '\n'; } diff --git a/clang/test/CodeGenCXX/nrvo.cpp b/clang/test/CodeGenCXX/nrvo.cpp --- a/clang/test/CodeGenCXX/nrvo.cpp +++ b/clang/test/CodeGenCXX/nrvo.cpp @@ -166,88 +166,19 @@ // CHECK-LABEL: @_Z5test3b( // CHECK-NEXT: entry: -// CHECK-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 -// CHECK-NEXT: br i1 [[B:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] -// CHECK: if.then: // CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) #[[ATTR5]] -// CHECK-NEXT: br label [[RETURN:%.*]] -// CHECK: if.end: -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_X]], %class.X* [[X]], i32 0, i32 0 -// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR5]] -// CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR5]] -// CHECK-NEXT: call void @_ZN1XC1ERKS_(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], %class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR5]] -// CHECK-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR5]] -// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR5]] -// CHECK-NEXT: br label [[RETURN]] -// CHECK: return: // CHECK-NEXT: ret void // -// CHECK-EH-03-LABEL: @_Z5test3b( -// CHECK-EH-03-NEXT: entry: -// CHECK-EH-03-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 -// CHECK-EH-03-NEXT: br i1 [[B:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] -// CHECK-EH-03: if.then: -// CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) -// CHECK-EH-03-NEXT: br label [[RETURN:%.*]] -// CHECK-EH-03: if.end: -// CHECK-EH-03-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_X]], %class.X* [[X]], i32 0, i32 0 -// CHECK-EH-03-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] -// CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], %class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] -// CHECK-EH-03: invoke.cont: -// CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] -// CHECK-EH-03-NEXT: br label [[RETURN]] -// CHECK-EH-03: lpad: -// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { i8*, i32 } -// CHECK-EH-03-NEXT: cleanup -// CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-03-NEXT: to label [[INVOKE_CONT1:%.*]] unwind label [[TERMINATE_LPAD:%.*]] -// CHECK-EH-03: invoke.cont1: -// CHECK-EH-03-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] -// CHECK-EH-03-NEXT: resume { i8*, i32 } [[TMP1]] -// CHECK-EH-03: return: -// CHECK-EH-03-NEXT: ret void -// CHECK-EH-03: terminate.lpad: -// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { i8*, i32 } -// CHECK-EH-03-NEXT: catch i8* null -// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { i8*, i32 } [[TMP2]], 0 -// CHECK-EH-03-NEXT: call void @__clang_call_terminate(i8* [[TMP3]]) #[[ATTR8]] -// CHECK-EH-03-NEXT: unreachable -// -// CHECK-EH-11-LABEL: @_Z5test3b( -// CHECK-EH-11-NEXT: entry: -// CHECK-EH-11-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 -// CHECK-EH-11-NEXT: br i1 [[B:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] -// CHECK-EH-11: if.then: -// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) -// CHECK-EH-11-NEXT: br label [[RETURN:%.*]] -// CHECK-EH-11: if.end: -// CHECK-EH-11-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_X]], %class.X* [[X]], i32 0, i32 0 -// CHECK-EH-11-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] -// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]], %class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) -// CHECK-EH-11-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] -// CHECK-EH-11: invoke.cont: -// CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR7]] -// CHECK-EH-11-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] -// CHECK-EH-11-NEXT: br label [[RETURN]] -// CHECK-EH-11: lpad: -// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { i8*, i32 } -// CHECK-EH-11-NEXT: cleanup -// CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR7]] -// CHECK-EH-11-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] -// CHECK-EH-11-NEXT: resume { i8*, i32 } [[TMP1]] -// CHECK-EH-11: return: -// CHECK-EH-11-NEXT: ret void +// CHECK-EH-LABEL: @_Z5test3b( +// CHECK-EH-NEXT: entry: +// CHECK-EH-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) +// CHECK-EH-NEXT: ret void // X test3(bool B) { if (B) { X y; return y; } - // FIXME: we should NRVO this variable too. X x; return x; } @@ -536,3 +467,146 @@ Y test9() { Y::f(); } + +// CHECK-LABEL: @_Z6test10b( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_X]], %class.X* [[X]], i32 0, i32 0 +// CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR5]] +// CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR5]] +// CHECK-NEXT: br i1 [[B:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +// CHECK: if.then: +// CHECK-NEXT: call void @_ZN1XC1ERKS_(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]], %class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR5]] +// CHECK-NEXT: br label [[CLEANUP:%.*]] +// CHECK: if.end: +// CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR5]] +// CHECK-NEXT: br label [[CLEANUP]] +// CHECK: cleanup: +// CHECK-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR5]] +// CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR5]] +// CHECK-NEXT: ret void +// +// CHECK-EH-03-LABEL: @_Z6test10b( +// CHECK-EH-03-NEXT: entry: +// CHECK-EH-03-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 +// CHECK-EH-03-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_X]], %class.X* [[X]], i32 0, i32 0 +// CHECK-EH-03-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] +// CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) +// CHECK-EH-03-NEXT: br i1 [[B:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +// CHECK-EH-03: if.then: +// CHECK-EH-03-NEXT: invoke void @_ZN1XC1ERKS_(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]], %class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) +// CHECK-EH-03-NEXT: to label [[CLEANUP:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-03: lpad: +// CHECK-EH-03-NEXT: [[TMP1:%.*]] = landingpad { i8*, i32 } +// CHECK-EH-03-NEXT: cleanup +// CHECK-EH-03-NEXT: invoke void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) +// CHECK-EH-03-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[TERMINATE_LPAD:%.*]] +// CHECK-EH-03: if.end: +// CHECK-EH-03-NEXT: invoke void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-03-NEXT: to label [[CLEANUP]] unwind label [[LPAD]] +// CHECK-EH-03: cleanup: +// CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) +// CHECK-EH-03-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] +// CHECK-EH-03-NEXT: ret void +// CHECK-EH-03: invoke.cont3: +// CHECK-EH-03-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] +// CHECK-EH-03-NEXT: resume { i8*, i32 } [[TMP1]] +// CHECK-EH-03: terminate.lpad: +// CHECK-EH-03-NEXT: [[TMP2:%.*]] = landingpad { i8*, i32 } +// CHECK-EH-03-NEXT: catch i8* null +// CHECK-EH-03-NEXT: [[TMP3:%.*]] = extractvalue { i8*, i32 } [[TMP2]], 0 +// CHECK-EH-03-NEXT: call void @__clang_call_terminate(i8* [[TMP3]]) #[[ATTR8]] +// CHECK-EH-03-NEXT: unreachable +// +// CHECK-EH-11-LABEL: @_Z6test10b( +// CHECK-EH-11-NEXT: entry: +// CHECK-EH-11-NEXT: [[X:%.*]] = alloca [[CLASS_X:%.*]], align 1 +// CHECK-EH-11-NEXT: [[TMP0:%.*]] = getelementptr inbounds [[CLASS_X]], %class.X* [[X]], i32 0, i32 0 +// CHECK-EH-11-NEXT: call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) +// CHECK-EH-11-NEXT: br i1 [[B:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +// CHECK-EH-11: if.then: +// CHECK-EH-11-NEXT: invoke void @_ZN1XC1ERKS_(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]], %class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) +// CHECK-EH-11-NEXT: to label [[CLEANUP:%.*]] unwind label [[LPAD:%.*]] +// CHECK-EH-11: lpad: +// CHECK-EH-11-NEXT: [[TMP1:%.*]] = landingpad { i8*, i32 } +// CHECK-EH-11-NEXT: cleanup +// CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: resume { i8*, i32 } [[TMP1]] +// CHECK-EH-11: if.end: +// CHECK-EH-11-NEXT: invoke void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-11-NEXT: to label [[CLEANUP]] unwind label [[LPAD]] +// CHECK-EH-11: cleanup: +// CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[X]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull [[TMP0]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: ret void +// +X test10(bool B) { + X x; // No NRVO + if (B) + return x; + X y; // NRVO + return y; +} + +// CHECK-LABEL: @_Z6test11bb( +// CHECK-NEXT: entry: +// CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) #[[ATTR5]] +// CHECK-NEXT: br i1 [[A:%.*]], label [[RETURN:%.*]], label [[NRVO_UNUSED:%.*]] +// CHECK: nrvo.unused: +// CHECK-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR5]] +// CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR5]] +// CHECK-NEXT: br i1 [[B:%.*]], label [[RETURN]], label [[NRVO_UNUSED8:%.*]] +// CHECK: nrvo.unused8: +// CHECK-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR5]] +// CHECK-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR5]] +// CHECK-NEXT: br label [[RETURN]] +// CHECK: return: +// CHECK-NEXT: ret void +// +// CHECK-EH-03-LABEL: @_Z6test11bb( +// CHECK-EH-03-NEXT: entry: +// CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) +// CHECK-EH-03-NEXT: br i1 [[A:%.*]], label [[RETURN:%.*]], label [[NRVO_UNUSED:%.*]] +// CHECK-EH-03: nrvo.unused: +// CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-03-NEXT: br i1 [[B:%.*]], label [[RETURN]], label [[NRVO_UNUSED8:%.*]] +// CHECK-EH-03: nrvo.unused8: +// CHECK-EH-03-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-03-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-03-NEXT: br label [[RETURN]] +// CHECK-EH-03: return: +// CHECK-EH-03-NEXT: ret void +// +// CHECK-EH-11-LABEL: @_Z6test11bb( +// CHECK-EH-11-NEXT: entry: +// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT:%.*]]) +// CHECK-EH-11-NEXT: br i1 [[A:%.*]], label [[RETURN:%.*]], label [[NRVO_UNUSED:%.*]] +// CHECK-EH-11: nrvo.unused: +// CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-11-NEXT: br i1 [[B:%.*]], label [[RETURN]], label [[NRVO_UNUSED8:%.*]] +// CHECK-EH-11: nrvo.unused8: +// CHECK-EH-11-NEXT: call void @_ZN1XD1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) #[[ATTR7]] +// CHECK-EH-11-NEXT: call void @_ZN1XC1Ev(%class.X* noundef nonnull align 1 dereferenceable(1) [[AGG_RESULT]]) +// CHECK-EH-11-NEXT: br label [[RETURN]] +// CHECK-EH-11: return: +// CHECK-EH-11-NEXT: ret void +// +X test11(bool A, bool B) { + // NRVO for each variable + { + { + X x; + if (A) + return x; + } + X y; + if (B) + return y; + } + X z; + return z; +}