Allow nested lambdas in NSDMIs (i.e default initializers)

[faisalv]

Summary

This patch fixes an infinite recursion issue when computing the linkage-visibility of nested lambdas in NSDMIs.
Allows compilation of this:
template<class T>

struct L {
    T t{};
    T t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
};
L<int> l;

}

Doug had glanced at this change in Chicago as part of my larger generic lambda patch - he thought it would be better addressed and reviewed as a separate patch (since the bug occurs with non-generic lambdas too).

Thanks!

[faisalv]

A slightly different approach - instead of finding the parent of the outermost lambda and checking for its linkage, first check if the outermost lambda has a mangling number, if not, return internal linkage for the nested lambda. If the outermost lambda does have a mangling number, then lets its parent determine its linkage/visibility. This approach doesn't break any of the recent codegen tests richard smith submitted for init-captures for lambdas.

[doug.gregor]

The code looks good, but please add a test that actually checks the computed linkage.

test/SemaCXX/lambda-expressions.cpp

270

We're not actually testing the linkage here. Please provide a test that generates IR and uses FileCheck to make sure the lambda gets the appropriate LLVM linkage.

[faisalv]

I added some IRgen tests to the code and then committed it - but the tests seemed to break a bunch of buildbots - so I reverted it - please help - what am i doing wrong?

[faisalv]

These IR tests should now work - ok to commit?

[doug.gregor]

Yep, LGTM.

lib/AST/Decl.cpp

                      LV.isVisibilityExplicit());
 }
 
+static inline const CXXRecordDecl*
+getOutermostEnclosingLambda(const CXXRecordDecl *Record) {
+  const CXXRecordDecl *Ret = Record;
+  while (Record && Record->isLambda()) {
+    Ret = Record;
+    if (!Record->getParent()) break;
+    // Get the Containing Class of this Lambda Class
+    Record = dyn_cast_or_null<CXXRecordDecl>(
+      Record->getParent()->getParent());
+  }
+  return Ret;
+}
+
 static LinkageInfo computeLVForDecl(const NamedDecl *D,
                                     LVComputationKind computation) {
   // Objective-C: treat all Objective-C declarations as having external

           return LinkageInfo::internal();
         }
 
-        // This lambda has its linkage/visibility determined by its owner.
-        return getLVForClosure(D->getDeclContext()->getRedeclContext(),
-                               Record->getLambdaContextDecl(), computation);
+        // This lambda has its linkage/visibility determined:
+        //  - either by the outermost lambda if that lambda has no mangling 
+        //    number. 
+        //  - or by the parent of the outer most lambda
+        // This prevents infinite recursion in settings such as nested lambdas 
+        // used in NSDMI's, for e.g. 
+        //  struct L {
+        //    int t{};
+        //    int t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
+        //  };
+        const CXXRecordDecl *OuterMostLambda = 
+            getOutermostEnclosingLambda(Record);
+        if (!OuterMostLambda->getLambdaManglingNumber())
+          return LinkageInfo::internal();
+        
+        return getLVForClosure(
+                  OuterMostLambda->getDeclContext()->getRedeclContext(),
+                  OuterMostLambda->getLambdaContextDecl(), computation);
       }
       
       break;

test/CodeGenCXX/lambda-expressions-nested-linkage.cpp

+// RUN: %clang_cc1 -triple x86_64-apple-darwin10.0.0 -fblocks -emit-llvm -o - %s -fexceptions -std=c++11 | FileCheck %s
+
+// CHECK-LABEL: define void @_ZN19non_inline_function3fooEv()
+// CHECK-LABEL: define internal void @"_ZZN19non_inline_function3fooEvENK3$_0clEi"(%class.anon
+// CHECK-LABEL: define internal signext i8 @"_ZZZN19non_inline_function3fooEvENK3$_0clEiENKUlcE_clEc"(%class.anon
+namespace non_inline_function {
+void foo() {
+  auto L = [](int a) {
+    return [](char b) {
+     return b;
+    };
+  };
+  L(3)('a');
+}
+}
+
+namespace non_template {
+  struct L {
+    int t = ([](int a) { return [](int b) { return b; };})(2)(3);    
+  };
+  L l; 
+}
+
+namespace lambdas_in_NSDMIs_template_class {
+template<class T>
+struct L {
+    T t2 = ([](int a) { return [](int b) { return b; };})(T{})(T{});    
+};
+L<int> l;
+}
+
+// CHECK-LABEL: define linkonce_odr i32 @_ZN15inline_function3fooEv
+// CHECK: define linkonce_odr void @_ZZN15inline_function3fooEvENKUliE_clEi
+// CHECK: define linkonce_odr signext i8 @_ZZZN15inline_function3fooEvENKUliE_clEiENKUlcE_clEc
+namespace inline_function {
+inline int foo() {
+  auto L = [](int a) {
+    return [](char b) {
+     return b;
+    };
+  };
+  L(3)('a');
+}
+int use = foo();
+}
+// CHECK: define linkonce_odr void @_ZNK32lambdas_in_NSDMIs_template_class1LIiEUliE_clEi(%class.anon
+// CHECK: define linkonce_odr i32 @_ZZNK32lambdas_in_NSDMIs_template_class1LIiEUliE_clEiENKUliE_clEi(%class.anon
+
+// CHECK: define linkonce_odr void @_ZNK12non_template1L1tMUliE_clEi(%class.anon
+// CHECK: define linkonce_odr i32 @_ZZNK12non_template1L1tMUliE_clEiENKUliE_clEi(%class.anon

test/SemaCXX/lambda-expressions.cpp

 #endif
   }
 }
+
+
+namespace lambdas_in_NSDMIs {
+  template<class T>
+  struct L {
+      T t{};
+      T t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
+  };
+  L<int> l; 
+  
+  namespace non_template {
+    struct L {
+      int t = 0;
+      int t2 = ([](int a) { return [](int b) { return b; };})(t)(t);    
+    };
+    L l; 
+  }
+}
 No newline at end of file