[CodeGen][ObjC] Fix infinite recursion in getObjCEncodingForTypeImpl

[ahatanak]

Summary

This patch fixes a stack overflow bug in ASTContext::getObjCEncodingForTypeImpl where it keeps expanding a class recursively. I added a check to avoid expanding a class if ExpandStructures is false.

[manmanren]

The fix looks reasonable to me.

Cheers,
Manman

test/CodeGenObjCXX/encode.mm

231

I wonder if we can further reduce the testing case. Is this necessary to cause the infinite recursion?

[ahatanak]

test/CodeGenObjCXX/encode.mm

231

Yes. If I remove the typedef and declare "t" as a pointer to "T", clang terminates normally. If you look at the code near ASTContext.cpp:5960, you'll see getObjCEncodingForTypeImpl is called recursively only if the pointee type is a typedef, which is why the typedef is necessary to cause infinite recursion in this case.

[manmanren]

LGTM.

Manman

lib/AST/ASTContext.cpp

void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string& S,
   }
   
   case Type::ObjCInterface: {
     // Ignore protocol qualifiers when mangling at this level.
     // @encode(class_name)
     ObjCInterfaceDecl *OI = T->castAs<ObjCObjectType>()->getInterface();
     S += '{';
     S += OI->getObjCRuntimeNameAsString();
+    if (ExpandStructures) {
       S += '=';
       SmallVector<const ObjCIvarDecl*, 32> Ivars;
       DeepCollectObjCIvars(OI, true, Ivars);
       for (unsigned i = 0, e = Ivars.size(); i != e; ++i) {
         const FieldDecl *Field = cast<FieldDecl>(Ivars[i]);
         if (Field->isBitField())
           getObjCEncodingForTypeImpl(Field->getType(), S, false, true, Field);
         else
           getObjCEncodingForTypeImpl(Field->getType(), S, false, true, FD,
                                      false, false, false, false, false,
                                      EncodePointerToObjCTypedef,
                                      NotEncodedT);
       }
+    }
     S += '}';
     return;
   }
 
   case Type::ObjCObjectPointer: {
     const ObjCObjectPointerType *OPT = T->castAs<ObjCObjectPointerType>();
     if (OPT->isObjCIdType()) {
       S += '@';

test/CodeGenObjCXX/encode.mm

namespace PR17142 {
   struct A { virtual ~A(); };
   struct B : virtual A { int y; };
   struct C { virtual ~C(); int z; };
   struct D : C, B { int a; };
   struct E : D {};
   // CHECK: @_ZN7PR171421xE = constant [14 x i8] c"{E=^^?i^^?ii}\00"
   extern const char x[] = @encode(E);
 }
+
+// This test used to cause infinite recursion.
+template<typename T>
+struct S {
+  typedef T Ty;
+  Ty *t;
+};
+
+@interface N
+{
+  S<N> a;
+}
+@end
+
+@implementation N
+@end
+
+const char *expand_struct() {
+  // CHECK: @{{.*}} = private unnamed_addr constant [16 x i8] c"{N={S<N>=^{N}}}\00"
+  return @encode(N);
+}