Index: include/llvm/Transforms/Utils/SimplifyLibCalls.h =================================================================== --- include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -37,11 +37,12 @@ /// is unknown) by passing true for OnlyLowerUnknownSize. class FortifiedLibCallSimplifier { private: + const DataLayout &DL; const TargetLibraryInfo *TLI; bool OnlyLowerUnknownSize; public: - FortifiedLibCallSimplifier(const TargetLibraryInfo *TLI, + FortifiedLibCallSimplifier(const DataLayout &DL, const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize = false); /// \brief Take the given call instruction and return a more Index: lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- lib/CodeGen/CodeGenPrepare.cpp +++ lib/CodeGen/CodeGenPrepare.cpp @@ -1863,7 +1863,7 @@ // to what InstCombineCalls does, but here we are only lowering calls // to fortified library functions (e.g. __memcpy_chk) that have the default // "don't know" as the objectsize. Anything else should be left alone. - FortifiedLibCallSimplifier Simplifier(TLInfo, true); + FortifiedLibCallSimplifier Simplifier(*DL, TLInfo, true); if (Value *V = Simplifier.optimizeCall(CI)) { CI->replaceAllUsesWith(V); CI->eraseFromParent(); Index: lib/Transforms/Utils/SimplifyLibCalls.cpp =================================================================== --- lib/Transforms/Utils/SimplifyLibCalls.cpp +++ lib/Transforms/Utils/SimplifyLibCalls.cpp @@ -915,12 +915,80 @@ return CI->getArgOperand(0); } +// TODO: Does this belong in BuildLibCalls or should all of those similar +// functions be moved here? +static Value *emitCalloc(Value *Num, Value *Size, const AttributeSet &Attrs, + IRBuilder<> &B, const DataLayout &DL, + const TargetLibraryInfo &TLI) { + if (!TLI.has(LibFunc::calloc)) + return nullptr; + + Module *M = B.GetInsertBlock()->getModule(); + IntegerType *PtrType = DL.getIntPtrType((B.GetInsertBlock()->getContext())); + Value *Calloc = M->getOrInsertFunction("calloc", Attrs, B.getInt8PtrTy(), + PtrType, PtrType, nullptr); + CallInst *CI = B.CreateCall(Calloc, { Num, Size }, "calloc"); + + if (const auto *F = dyn_cast(Calloc->stripPointerCasts())) + CI->setCallingConv(F->getCallingConv()); + + return CI; +} + +/// Fold memset[_chk](malloc(n), 0, n) --> calloc(1, n). +static Value *foldMallocMemset(CallInst *Memset, IRBuilder<> &B, + const TargetLibraryInfo &TLI, + const DataLayout &DL) { + // This has to be a memset of zeros (bzero). + auto *FillValue = dyn_cast(Memset->getArgOperand(1)); + if (!FillValue || FillValue->getZExtValue() != 0) + return nullptr; + + auto *Malloc = dyn_cast(Memset->getArgOperand(0)); + if (!Malloc) + return nullptr; + + // Is the inner call really malloc()? + Function *InnerCallee = Malloc->getCalledFunction(); + LibFunc::Func Func; + if (!TLI.getLibFunc(InnerCallee->getName(), Func) || !TLI.has(Func) || + Func != LibFunc::malloc) + return nullptr; + + // Matching the name is not good enough. Make sure the parameter and return + // type match the standard library signature. + FunctionType *FT = InnerCallee->getFunctionType(); + if (FT->getNumParams() != 1 || !FT->getParamType(0)->isIntegerTy()) + return nullptr; + + auto *RetType = dyn_cast(FT->getReturnType()); + if (!RetType || !RetType->getPointerElementType()->isIntegerTy(8)) + return nullptr; + + // The memset must cover the same number of bytes that are malloc'd. + if (Memset->getArgOperand(2) != Malloc->getArgOperand(0)) + return nullptr; + + // Replace the malloc with a calloc. + B.SetInsertPoint(Malloc->getParent(), ++Malloc->getIterator()); + IntegerType *SizeType = DL.getIntPtrType(B.GetInsertBlock()->getContext()); + Value *Calloc = emitCalloc(ConstantInt::get(SizeType, 1), + Malloc->getArgOperand(0), Malloc->getAttributes(), + B, DL, TLI); + Malloc->replaceAllUsesWith(Calloc); + Malloc->eraseFromParent(); + + return Calloc; +} + Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilder<> &B) { Function *Callee = CI->getCalledFunction(); if (!checkStringCopyLibFuncSignature(Callee, LibFunc::memset)) return nullptr; + // TODO: Use foldMallocMemset() here. + // memset(p, v, n) -> llvm.memset(p, v, n, 1) Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false); B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1); @@ -2188,6 +2256,7 @@ return optimizeLog(CI, Builder); case Intrinsic::sqrt: return optimizeSqrt(CI, Builder); + // TODO: Use foldMallocMemset() with memset intrinsic. default: return nullptr; } @@ -2334,7 +2403,7 @@ LibCallSimplifier::LibCallSimplifier( const DataLayout &DL, const TargetLibraryInfo *TLI, function_ref Replacer) - : FortifiedSimplifier(TLI), DL(DL), TLI(TLI), UnsafeFPShrink(false), + : FortifiedSimplifier(DL, TLI), DL(DL), TLI(TLI), UnsafeFPShrink(false), Replacer(Replacer) {} void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) { @@ -2451,6 +2520,9 @@ if (!checkStringCopyLibFuncSignature(Callee, LibFunc::memset_chk)) return nullptr; + if (Value *Calloc = foldMallocMemset(CI, B, *TLI, DL)) + return Calloc; + if (isFortifiedCallFoldable(CI, 3, 2, false)) { Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false); B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1); @@ -2571,5 +2643,5 @@ } FortifiedLibCallSimplifier::FortifiedLibCallSimplifier( - const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize) - : TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {} + const DataLayout &DL, const TargetLibraryInfo *TLI, bool OnlyLowerUnknownSize) + : DL(DL), TLI(TLI), OnlyLowerUnknownSize(OnlyLowerUnknownSize) {} Index: test/Transforms/InstCombine/memset_chk-1.ll =================================================================== --- test/Transforms/InstCombine/memset_chk-1.ll +++ test/Transforms/InstCombine/memset_chk-1.ll @@ -91,8 +91,6 @@ declare i8* @__memset_chk(i8*, i32, i64, i64) -; FIXME: memset(malloc(x), 0, x) -> calloc(1, x) - define float* @pr25892(i64 %size) #0 { entry: %call = tail call i8* @malloc(i64 %size) #1 @@ -109,13 +107,11 @@ ; CHECK-LABEL: @pr25892( ; CHECK: entry: -; CHECK-NEXT: %call = tail call i8* @malloc(i64 %size) -; CHECK-NEXT: %cmp = icmp eq i8* %call, null +; CHECK-NEXT: %calloc = call i8* @calloc(i64 1, i64 %size) +; CHECK-NEXT: %cmp = icmp eq i8* %calloc, null ; CHECK-NEXT: br i1 %cmp, label %cleanup, label %if.end ; CHECK: if.end: -; CHECK-NEXT: %bc = bitcast i8* %call to float* -; CHECK-NEXT: %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false) -; CHECK-NEXT: %call3 = tail call i8* @__memset_chk(i8* nonnull %call, i32 0, i64 %size, i64 %call2) +; CHECK-NEXT: %bc = bitcast i8* %calloc to float* ; CHECK-NEXT: br label %cleanup ; CHECK: cleanup: ; CHECK-NEXT: %retval.0 = phi float* [ %bc, %if.end ], [ null, %entry ]