Index: llvm/trunk/docs/LangRef.rst =================================================================== --- llvm/trunk/docs/LangRef.rst +++ llvm/trunk/docs/LangRef.rst @@ -8314,7 +8314,7 @@ :: - = [tail | musttail | notail ] call [cconv] [ret attrs] [*] () [fn attrs] + = [tail | musttail | notail ] call [fast-math flags] [cconv] [ret attrs] [*] () [fn attrs] [ operand bundles ] Overview: @@ -8371,6 +8371,11 @@ ``tail`` or ``musttail`` markers to the call. It is used to prevent tail call optimization from being performed on the call. +#. The optional ``fast-math flags`` marker indicates that the call has one or more + :ref:`fast-math flags `, which are optimization hints to enable + otherwise unsafe floating-point optimizations. Fast-math flags are only valid + for calls that return a floating-point scalar or vector type. + #. The optional "cconv" marker indicates which :ref:`calling convention ` the call should use. If none is specified, the call defaults to using C calling conventions. The Index: llvm/trunk/include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- llvm/trunk/include/llvm/Bitcode/LLVMBitCodes.h +++ llvm/trunk/include/llvm/Bitcode/LLVMBitCodes.h @@ -348,7 +348,8 @@ CALL_CCONV = 1, CALL_MUSTTAIL = 14, CALL_EXPLICIT_TYPE = 15, - CALL_NOTAIL = 16 + CALL_NOTAIL = 16, + CALL_FMF = 17 // Call has optional fast-math-flags. }; // The function body block (FUNCTION_BLOCK_ID) describes function bodies. It Index: llvm/trunk/include/llvm/IR/IRBuilder.h =================================================================== --- llvm/trunk/include/llvm/IR/IRBuilder.h +++ llvm/trunk/include/llvm/IR/IRBuilder.h @@ -1532,19 +1532,26 @@ const Twine &Name = "") { return Insert(CallInst::Create(Callee, Args, OpBundles), Name); } + CallInst *CreateCall(Value *Callee, ArrayRef Args, - const Twine &Name) { - return Insert(CallInst::Create(Callee, Args), Name); + const Twine &Name, MDNode *FPMathTag = nullptr) { + PointerType *PTy = cast(Callee->getType()); + FunctionType *FTy = cast(PTy->getElementType()); + return CreateCall(FTy, Callee, Args, Name, FPMathTag); } CallInst *CreateCall(llvm::FunctionType *FTy, Value *Callee, - ArrayRef Args, const Twine &Name = "") { - return Insert(CallInst::Create(FTy, Callee, Args), Name); + ArrayRef Args, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + CallInst *CI = CallInst::Create(FTy, Callee, Args); + if (isa(CI)) + CI = cast(AddFPMathAttributes(CI, FPMathTag, FMF)); + return Insert(CI, Name); } CallInst *CreateCall(Function *Callee, ArrayRef Args, - const Twine &Name = "") { - return CreateCall(Callee->getFunctionType(), Callee, Args, Name); + const Twine &Name = "", MDNode *FPMathTag = nullptr) { + return CreateCall(Callee->getFunctionType(), Callee, Args, Name, FPMathTag); } Value *CreateSelect(Value *C, Value *True, Value *False, Index: llvm/trunk/lib/AsmParser/LLParser.cpp =================================================================== --- llvm/trunk/lib/AsmParser/LLParser.cpp +++ llvm/trunk/lib/AsmParser/LLParser.cpp @@ -5603,14 +5603,14 @@ } /// ParseCall -/// ::= 'call' OptionalCallingConv OptionalAttrs Type Value -/// ParameterList OptionalAttrs -/// ::= 'tail' 'call' OptionalCallingConv OptionalAttrs Type Value -/// ParameterList OptionalAttrs -/// ::= 'musttail' 'call' OptionalCallingConv OptionalAttrs Type Value -/// ParameterList OptionalAttrs -/// ::= 'notail' 'call' OptionalCallingConv OptionalAttrs Type Value -/// ParameterList OptionalAttrs +/// ::= 'call' OptionalFastMathFlags OptionalCallingConv +/// OptionalAttrs Type Value ParameterList OptionalAttrs +/// ::= 'tail' 'call' OptionalFastMathFlags OptionalCallingConv +/// OptionalAttrs Type Value ParameterList OptionalAttrs +/// ::= 'musttail' 'call' OptionalFastMathFlags OptionalCallingConv +/// OptionalAttrs Type Value ParameterList OptionalAttrs +/// ::= 'notail' 'call' OptionalFastMathFlags OptionalCallingConv +/// OptionalAttrs Type Value ParameterList OptionalAttrs bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS, CallInst::TailCallKind TCK) { AttrBuilder RetAttrs, FnAttrs; @@ -5624,10 +5624,14 @@ SmallVector BundleList; LocTy CallLoc = Lex.getLoc(); - if ((TCK != CallInst::TCK_None && - ParseToken(lltok::kw_call, - "expected 'tail call', 'musttail call', or 'notail call'")) || - ParseOptionalCallingConv(CC) || ParseOptionalReturnAttrs(RetAttrs) || + if (TCK != CallInst::TCK_None && + ParseToken(lltok::kw_call, + "expected 'tail call', 'musttail call', or 'notail call'")) + return true; + + FastMathFlags FMF = EatFastMathFlagsIfPresent(); + + if (ParseOptionalCallingConv(CC) || ParseOptionalReturnAttrs(RetAttrs) || ParseType(RetType, RetTypeLoc, true /*void allowed*/) || ParseValID(CalleeID) || ParseParameterList(ArgList, PFS, TCK == CallInst::TCK_MustTail, @@ -5636,6 +5640,10 @@ ParseOptionalOperandBundles(BundleList, PFS)) return true; + if (FMF.any() && !RetType->isFPOrFPVectorTy()) + return Error(CallLoc, "fast-math-flags specified for call without " + "floating-point scalar or vector return type"); + // If RetType is a non-function pointer type, then this is the short syntax // for the call, which means that RetType is just the return type. Infer the // rest of the function argument types from the arguments that are present. @@ -5708,6 +5716,8 @@ CallInst *CI = CallInst::Create(Ty, Callee, Args, BundleList); CI->setTailCallKind(TCK); CI->setCallingConv(CC); + if (FMF.any()) + CI->setFastMathFlags(FMF); CI->setAttributes(PAL); ForwardRefAttrGroups[CI] = FwdRefAttrGrps; Inst = CI; Index: llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp +++ llvm/trunk/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5006,7 +5006,7 @@ break; } case bitc::FUNC_CODE_INST_CALL: { - // CALL: [paramattrs, cc, fnty, fnid, arg0, arg1...] + // CALL: [paramattrs, cc, fmf, fnty, fnid, arg0, arg1...] if (Record.size() < 3) return error("Invalid record"); @@ -5014,6 +5014,13 @@ AttributeSet PAL = getAttributes(Record[OpNum++]); unsigned CCInfo = Record[OpNum++]; + FastMathFlags FMF; + if ((CCInfo >> bitc::CALL_FMF) & 1) { + FMF = getDecodedFastMathFlags(Record[OpNum++]); + if (!FMF.any()) + return error("Fast math flags indicator set for call with no FMF"); + } + FunctionType *FTy = nullptr; if (CCInfo >> bitc::CALL_EXPLICIT_TYPE & 1 && !(FTy = dyn_cast(getTypeByID(Record[OpNum++])))) @@ -5075,6 +5082,12 @@ TCK = CallInst::TCK_NoTail; cast(I)->setTailCallKind(TCK); cast(I)->setAttributes(PAL); + if (FMF.any()) { + if (!isa(I)) + return error("Fast-math-flags specified for call without " + "floating-point scalar or vector return type"); + I->setFastMathFlags(FMF); + } break; } case bitc::FUNC_CODE_INST_VAARG: { // VAARG: [valistty, valist, instty] Index: llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp +++ llvm/trunk/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -2153,11 +2153,17 @@ Code = bitc::FUNC_CODE_INST_CALL; Vals.push_back(VE.getAttributeID(CI.getAttributes())); + + unsigned Flags = GetOptimizationFlags(&I); Vals.push_back(CI.getCallingConv() << bitc::CALL_CCONV | unsigned(CI.isTailCall()) << bitc::CALL_TAIL | unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL | 1 << bitc::CALL_EXPLICIT_TYPE | - unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL); + unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL | + unsigned(Flags != 0) << bitc::CALL_FMF); + if (Flags != 0) + Vals.push_back(Flags); + Vals.push_back(VE.getTypeID(FTy)); PushValueAndType(CI.getCalledValue(), InstID, Vals, VE); // Callee Index: llvm/trunk/test/Bitcode/compatibility.ll =================================================================== --- llvm/trunk/test/Bitcode/compatibility.ll +++ llvm/trunk/test/Bitcode/compatibility.ll @@ -664,6 +664,28 @@ ret void } +; Check various fast math flags and floating-point types on calls. + +declare float @fmf1() +declare double @fmf2() +declare <4 x double> @fmf3() + +; CHECK-LABEL: fastMathFlagsForCalls( +define void @fastMathFlagsForCalls(float %f, double %d1, <4 x double> %d2) { + %call.fast = call fast float @fmf1() + ; CHECK: %call.fast = call fast float @fmf1() + + ; Throw in some other attributes to make sure those stay in the right places. + + %call.nsz.arcp = notail call nsz arcp double @fmf2() + ; CHECK: %call.nsz.arcp = notail call nsz arcp double @fmf2() + + %call.nnan.ninf = tail call nnan ninf fastcc <4 x double> @fmf3() + ; CHECK: %call.nnan.ninf = tail call nnan ninf fastcc <4 x double> @fmf3() + + ret void +} + ;; Type System %opaquety = type opaque define void @typesystem() { Index: llvm/trunk/test/Transforms/InstCombine/fast-math.ll =================================================================== --- llvm/trunk/test/Transforms/InstCombine/fast-math.ll +++ llvm/trunk/test/Transforms/InstCombine/fast-math.ll @@ -570,7 +570,7 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_arg_squared( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) ; CHECK-NEXT: ret double %fabs } @@ -584,8 +584,8 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_three_args1( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %y) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %y) ; CHECK-NEXT: %1 = fmul fast double %fabs, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -597,8 +597,8 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_three_args2( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %y) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %y) ; CHECK-NEXT: %1 = fmul fast double %fabs, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -610,8 +610,8 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_three_args3( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %y) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %y) ; CHECK-NEXT: %1 = fmul fast double %fabs, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -623,8 +623,8 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_three_args4( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %y) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %y) ; CHECK-NEXT: %1 = fmul fast double %fabs, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -636,8 +636,8 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_three_args5( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %y) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %y) ; CHECK-NEXT: %1 = fmul fast double %fabs, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -649,8 +649,8 @@ ret double %sqrt ; CHECK-LABEL: sqrt_intrinsic_three_args6( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %y) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %y) ; CHECK-NEXT: %1 = fmul fast double %fabs, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -675,7 +675,7 @@ ; CHECK-LABEL: sqrt_intrinsic_arg_5th( ; CHECK-NEXT: %mul = fmul fast double %x, %x -; CHECK-NEXT: %sqrt1 = call double @llvm.sqrt.f64(double %x) +; CHECK-NEXT: %sqrt1 = call fast double @llvm.sqrt.f64(double %x) ; CHECK-NEXT: %1 = fmul fast double %mul, %sqrt1 ; CHECK-NEXT: ret double %1 } @@ -692,7 +692,7 @@ ret float %sqrt ; CHECK-LABEL: sqrt_call_squared_f32( -; CHECK-NEXT: %fabs = call float @llvm.fabs.f32(float %x) +; CHECK-NEXT: %fabs = call fast float @llvm.fabs.f32(float %x) ; CHECK-NEXT: ret float %fabs } @@ -702,7 +702,7 @@ ret double %sqrt ; CHECK-LABEL: sqrt_call_squared_f64( -; CHECK-NEXT: %fabs = call double @llvm.fabs.f64(double %x) +; CHECK-NEXT: %fabs = call fast double @llvm.fabs.f64(double %x) ; CHECK-NEXT: ret double %fabs } @@ -712,7 +712,7 @@ ret fp128 %sqrt ; CHECK-LABEL: sqrt_call_squared_f128( -; CHECK-NEXT: %fabs = call fp128 @llvm.fabs.f128(fp128 %x) +; CHECK-NEXT: %fabs = call fast fp128 @llvm.fabs.f128(fp128 %x) ; CHECK-NEXT: ret fp128 %fabs } Index: llvm/trunk/test/Transforms/InstCombine/inline-intrinsic-assert.ll =================================================================== --- llvm/trunk/test/Transforms/InstCombine/inline-intrinsic-assert.ll +++ llvm/trunk/test/Transforms/InstCombine/inline-intrinsic-assert.ll @@ -9,7 +9,7 @@ ret float %call ; CHECK-LABEL: @foo( -; CHECK-NEXT: call float @llvm.fabs.f32 +; CHECK-NEXT: call fast float @llvm.fabs.f32 ; CHECK-NEXT: ret float } Index: llvm/trunk/test/Transforms/InstCombine/log-pow.ll =================================================================== --- llvm/trunk/test/Transforms/InstCombine/log-pow.ll +++ llvm/trunk/test/Transforms/InstCombine/log-pow.ll @@ -8,7 +8,7 @@ } ; CHECK-LABEL: define double @mylog( -; CHECK: %log = call double @log(double %x) #0 +; CHECK: %log = call fast double @log(double %x) #0 ; CHECK: %mul = fmul fast double %log, %y ; CHECK: ret double %mul ; CHECK: } Index: llvm/trunk/test/Transforms/InstCombine/no_cgscc_assert.ll =================================================================== --- llvm/trunk/test/Transforms/InstCombine/no_cgscc_assert.ll +++ llvm/trunk/test/Transforms/InstCombine/no_cgscc_assert.ll @@ -10,7 +10,7 @@ ret float %call1 ; CHECK-LABEL: @bar( -; CHECK-NEXT: call float @llvm.fabs.f32 +; CHECK-NEXT: call fast float @llvm.fabs.f32 ; CHECK-NEXT: ret float } Index: llvm/trunk/test/Transforms/InstCombine/pow-exp.ll =================================================================== --- llvm/trunk/test/Transforms/InstCombine/pow-exp.ll +++ llvm/trunk/test/Transforms/InstCombine/pow-exp.ll @@ -9,7 +9,7 @@ ; CHECK-LABEL: define double @mypow( ; CHECK: %mul = fmul fast double %x, %y -; CHECK: %exp = call double @exp(double %mul) #0 +; CHECK: %exp = call fast double @exp(double %mul) #0 ; CHECK: ret double %exp ; CHECK: } Index: llvm/trunk/test/Transforms/InstCombine/pow-exp2.ll =================================================================== --- llvm/trunk/test/Transforms/InstCombine/pow-exp2.ll +++ llvm/trunk/test/Transforms/InstCombine/pow-exp2.ll @@ -9,7 +9,7 @@ ; CHECK-LABEL: define double @mypow( ; CHECK: %mul = fmul fast double %x, %y -; CHECK: %exp2 = call double @exp2(double %mul) #0 +; CHECK: %exp2 = call fast double @exp2(double %mul) #0 ; CHECK: ret double %exp2 ; CHECK: } Index: llvm/trunk/unittests/IR/IRBuilderTest.cpp =================================================================== --- llvm/trunk/unittests/IR/IRBuilderTest.cpp +++ llvm/trunk/unittests/IR/IRBuilderTest.cpp @@ -131,7 +131,7 @@ TEST_F(IRBuilderTest, FastMathFlags) { IRBuilder<> Builder(BB); Value *F, *FC; - Instruction *FDiv, *FAdd, *FCmp; + Instruction *FDiv, *FAdd, *FCmp, *FCall; F = Builder.CreateLoad(GV); F = Builder.CreateFAdd(F, F); @@ -207,6 +207,26 @@ EXPECT_TRUE(FCmp->hasAllowReciprocal()); Builder.clearFastMathFlags(); + + // Test a call with FMF. + auto CalleeTy = FunctionType::get(Type::getFloatTy(Ctx), + /*isVarArg=*/false); + auto Callee = + Function::Create(CalleeTy, Function::ExternalLinkage, "", M.get()); + + FCall = Builder.CreateCall(Callee, None); + EXPECT_FALSE(FCall->hasNoNaNs()); + + FMF.clear(); + FMF.setNoNaNs(); + Builder.SetFastMathFlags(FMF); + + FCall = Builder.CreateCall(Callee, None); + EXPECT_TRUE(Builder.getFastMathFlags().any()); + EXPECT_TRUE(Builder.getFastMathFlags().NoNaNs); + EXPECT_TRUE(FCall->hasNoNaNs()); + + Builder.clearFastMathFlags(); // To test a copy, make sure that a '0' and a '1' change state. F = Builder.CreateFDiv(F, F);