Fix a suspicious check in TargetLibraryInfo

[ab]

Summary

The check for some of the builtin functions involves checking if the number
of the parameter is zero and the first parameter is a pointer type. This check
either returns false or an out-of-bound memory access.

rdar://problem/26424030

[ab]

Looks good, but can you test the other changed functions? If you'd like, I can take care of it.

[ab]

• Test everything

[steven_wu]

Thanks for adding all the tests! LGTM

lib/Analysis/TargetLibraryInfo.cpp

return (NumParams == 3 && FTy.getReturnType() == FTy.getParamType(0) &&
             FTy.getParamType(1)->isIntegerTy() &&
             IsSizeTTy(FTy.getParamType(2)));
 
   case LibFunc::memccpy:
     return (NumParams >= 2 && FTy.getParamType(1)->isPointerTy());
   case LibFunc::memalign:
     return (FTy.getReturnType()->isPointerTy());
   case LibFunc::mkdir:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::mktime:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
+    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::realloc:
     return (NumParams == 2 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getReturnType()->isPointerTy());
   case LibFunc::read:
     return (NumParams == 3 && FTy.getParamType(1)->isPointerTy());
   case LibFunc::rewind:
     return (NumParams >= 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::rmdir:
case LibFunc::bcmp:
     return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::bzero:
     return (NumParams == 2 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::calloc:
     return (NumParams == 2 && FTy.getReturnType()->isPointerTy());
   case LibFunc::chmod:
   case LibFunc::chown:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::ctermid:
   case LibFunc::clearerr:
   case LibFunc::closedir:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
+    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::atoi:
   case LibFunc::atol:
   case LibFunc::atof:
   case LibFunc::atoll:
     return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::access:
     return (NumParams == 2 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::fopen:
bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
   case LibFunc::ftello:
   case LibFunc::fileno:
   case LibFunc::fflush:
   case LibFunc::fclose:
   case LibFunc::fsetpos:
   case LibFunc::flockfile:
   case LibFunc::funlockfile:
   case LibFunc::ftrylockfile:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::ferror:
     return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::fputc:
   case LibFunc::fstat:
   case LibFunc::frexp:
   case LibFunc::frexpf:
   case LibFunc::frexpl:
   case LibFunc::fstatvfs:
bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
   case LibFunc::fscanf:
   case LibFunc::fprintf:
     return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::fgetpos:
     return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::getc:
+    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::getlogin_r:
+    return (NumParams == 2 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::getc_unlocked:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::getenv:
     return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::gets:
   case LibFunc::getchar:
   case LibFunc::getitimer:
     return (NumParams == 2 && FTy.getParamType(1)->isPointerTy());
   case LibFunc::getpwnam:
     return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
case LibFunc::dunder_isoc99_sscanf:
     return (NumParams >= 1 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::fopen64:
     return (NumParams == 2 && FTy.getReturnType()->isPointerTy() &&
             FTy.getParamType(0)->isPointerTy() &&
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::fseeko64:
   case LibFunc::ftello64:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
+    return (NumParams == 3 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::tmpfile64:
     return (FTy.getReturnType()->isPointerTy());
   case LibFunc::fstat64:
   case LibFunc::fstatvfs64:
     return (NumParams == 2 && FTy.getParamType(1)->isPointerTy());
   case LibFunc::open64:
     return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::gettimeofday:

test/Transforms/InferFunctionAttrs/annotate.ll

 ; CHECK: attributes [[G0]] = { nounwind }
 ; CHECK: attributes [[G1]] = { nounwind readonly }
 ; CHECK-POSIX: attributes [[G0]] = { nounwind }
 ; CHECK-POSIX: attributes [[G2]] = { argmemonly }
 
 declare i32 @__nvvm_reflect(i8*)
 ; CHECK-NVPTX: declare i32 @__nvvm_reflect(i8*) [[G0:#[0-9]+]]
 ; CHECK-NVPTX: attributes [[G0]] = { nounwind readnone }
+
+; Make sure following testcase doesn't crash especially with ASAN
+declare void @free(...)