Index: docs/LanguageExtensions.rst =================================================================== --- docs/LanguageExtensions.rst +++ docs/LanguageExtensions.rst @@ -2312,3 +2312,32 @@ proven safe to vectorize. To identify and diagnose optimization issues use `-Rpass`, `-Rpass-missed`, and `-Rpass-analysis` command line options. See the user guide for details. + +Extensions to specify floating-point fast-math flags +==================================================== + +The ``#pragma clang fast_math`` pragma allows floating-point fast-math options +to be specified for a section of the source code. This pragma can only appear +at file scope or at the start of a compound statement (excluding +comments). When using within a compound statement, the pragma is active within +the scope of the compound statement. + +Currently, only FP contraction can be controlled with the pragma. ``#pragma +clang fast_math contract_fast(on)`` enables contraction of a multiply and an +addition (or subtraction) into a fused FMA operation when supported by the +target. + +.. code-block:: c++ + + for(...) { + #pragma clang fast_math contract_fast(on) + a = b[i] * c[i]; + d[i] += a; + } + +The pragma is similar to ``#pragma STDC FP_CONTRACT`` but it also allows +fusion in cases where the language standard does not make this possible. + +The pragma can also be used with 'off' which turns FP contraction off for a +section of the code. This can be useful when fast contraction is otherwise +enabled for the translation unit with the ``-ffp-contract=fast` flag. Index: include/clang/Basic/DiagnosticParseKinds.td =================================================================== --- include/clang/Basic/DiagnosticParseKinds.td +++ include/clang/Basic/DiagnosticParseKinds.td @@ -1042,6 +1042,16 @@ def err_pragma_loop_invalid_option : Error< "%select{invalid|missing}0 option%select{ %1|}0; expected vectorize, " "vectorize_width, interleave, interleave_count, unroll, unroll_count, or distribute">; + +def err_pragma_fast_math_invalid_option : Error< + "%select{invalid|missing}0 option%select{ %1|}0; expected contract_fast">; +def err_pragma_fast_math_invalid_argument : Error< + "unexpected argument '%0' to '#pragma clang fast_math %1'; " + "expected 'on' or 'off'">; +def err_pragma_fast_math_scope : Error< + "'#pragma clang fast_math' can only appear at file scope or at the start of a " + "compound statement">; + def err_pragma_invalid_keyword : Error< "invalid argument; expected 'enable'%select{|, 'full'}0%select{|, 'assume_safety'}1 or 'disable'">; Index: include/clang/Basic/TokenKinds.def =================================================================== --- include/clang/Basic/TokenKinds.def +++ include/clang/Basic/TokenKinds.def @@ -787,6 +787,8 @@ // handles #pragma loop ... directives. ANNOTATION(pragma_loop_hint) +ANNOTATION(pragma_fast_math) + // Annotations for module import translated from #include etc. ANNOTATION(module_include) ANNOTATION(module_begin) Index: include/clang/Parse/Parser.h =================================================================== --- include/clang/Parse/Parser.h +++ include/clang/Parse/Parser.h @@ -183,6 +183,7 @@ std::unique_ptr LoopHintHandler; std::unique_ptr UnrollHintHandler; std::unique_ptr NoUnrollHintHandler; + std::unique_ptr FastMathHandler; std::unique_ptr CommentSemaHandler; @@ -547,6 +548,7 @@ /// \brief Handle the annotation token produced for /// #pragma STDC FP_CONTRACT... void HandlePragmaFPContract(); + void HandlePragmaFastMath(); /// \brief Handle the annotation token produced for /// #pragma OPENCL EXTENSION... Index: include/clang/Sema/Sema.h =================================================================== --- include/clang/Sema/Sema.h +++ include/clang/Sema/Sema.h @@ -8112,8 +8112,9 @@ SourceLocation AliasNameLoc); /// ActOnPragmaFPContract - Called on well formed - /// \#pragma {STDC,OPENCL} FP_CONTRACT - void ActOnPragmaFPContract(tok::OnOffSwitch OOS); + /// \#pragma {STDC,OPENCL} FP_CONTRACT and + /// \#pragma clang fast_math contract_fast + void ActOnPragmaFPContract(LangOptions::FPContractModeKind FPC); /// AddAlignmentAttributesForRecord - Adds any needed alignment attributes to /// a the record decl, to handle '\#pragma pack' and '\#pragma options align'. Index: lib/Parse/ParsePragma.cpp =================================================================== --- lib/Parse/ParsePragma.cpp +++ lib/Parse/ParsePragma.cpp @@ -86,6 +86,12 @@ Token &FirstToken) override; }; +struct PragmaFastMathHandler : public PragmaHandler { + PragmaFastMathHandler() : PragmaHandler("fast_math") {} + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &FirstToken) override; +}; + struct PragmaNoOpenMPHandler : public PragmaHandler { PragmaNoOpenMPHandler() : PragmaHandler("omp") { } void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, @@ -266,6 +272,9 @@ NoUnrollHintHandler.reset(new PragmaUnrollHintHandler("nounroll")); PP.AddPragmaHandler(NoUnrollHintHandler.get()); + + FastMathHandler.reset(new PragmaFastMathHandler()); + PP.AddPragmaHandler("clang", FastMathHandler.get()); } void Parser::resetPragmaHandlers() { @@ -344,6 +353,9 @@ PP.RemovePragmaHandler(NoUnrollHintHandler.get()); NoUnrollHintHandler.reset(); + + PP.RemovePragmaHandler("clang", FastMathHandler.get()); + FastMathHandler.reset(); } /// \brief Handle the annotation token produced for #pragma unused(...) @@ -454,7 +466,21 @@ tok::OnOffSwitch OOS = static_cast( reinterpret_cast(Tok.getAnnotationValue())); - Actions.ActOnPragmaFPContract(OOS); + + LangOptions::FPContractModeKind FPC; + switch (OOS) { + case tok::OOS_ON: + FPC = LangOptions::FPC_On; + break; + case tok::OOS_OFF: + FPC = LangOptions::FPC_Off; + break; + case tok::OOS_DEFAULT: + FPC = getLangOpts().getDefaultFPContractMode(); + break; + } + + Actions.ActOnPragmaFPContract(FPC); ConsumeToken(); // The annotation token. } @@ -1947,6 +1973,113 @@ Actions.ActOnPragmaOptimize(IsOn, FirstToken.getLocation()); } +namespace { +/// Used as the annotation value for tok::annot_pragma_fast_math. +struct TokFastMathAnnotValue { + enum FlagKinds { ContractFast }; + + FlagKinds FlagKind; + bool FlagValue; +}; +} // end anonymous namespace + +void PragmaFastMathHandler::HandlePragma(Preprocessor &PP, + PragmaIntroducerKind Introducer, + Token &Tok) { + // fast_math + Token PragmaName = Tok; + SmallVector TokenList; + + PP.Lex(Tok); + if (Tok.isNot(tok::identifier)) { + PP.Diag(Tok.getLocation(), diag::err_pragma_fast_math_invalid_option) + << /*MissingOption=*/true << ""; + return; + } + + while (Tok.is(tok::identifier)) { + IdentifierInfo *OptionInfo = Tok.getIdentifierInfo(); + + auto FlagKind = + llvm::StringSwitch>( + OptionInfo->getName()) + .Case("contract_fast", TokFastMathAnnotValue::ContractFast) + .Default(None); + if (!FlagKind) { + PP.Diag(Tok.getLocation(), diag::err_pragma_fast_math_invalid_option) + << /*MissingOption=*/false << OptionInfo; + return; + } + PP.Lex(Tok); + + // Read '(' + if (Tok.isNot(tok::l_paren)) { + PP.Diag(Tok.getLocation(), diag::err_expected) << tok::l_paren; + return; + } + PP.Lex(Tok); + + if (Tok.isNot(tok::identifier)) { + PP.Diag(Tok.getLocation(), diag::err_pragma_fast_math_invalid_argument) + << PP.getSpelling(Tok) << OptionInfo->getName(); + return; + } + const IdentifierInfo *II = Tok.getIdentifierInfo(); + + auto FlagValue = llvm::StringSwitch>(II->getName()) + .Case("on", true) + .Case("off", false) + .Default(llvm::None); + + if (!FlagValue) { + PP.Diag(Tok.getLocation(), diag::err_pragma_fast_math_invalid_argument) + << PP.getSpelling(Tok) << OptionInfo->getName(); + return; + } + PP.Lex(Tok); + + // Read ')' + if (Tok.isNot(tok::r_paren)) { + PP.Diag(Tok.getLocation(), diag::err_expected) << tok::r_paren; + return; + } + PP.Lex(Tok); + + auto *AnnotValue = new (PP.getPreprocessorAllocator()) + TokFastMathAnnotValue{*FlagKind, *FlagValue}; + // Generate the loop hint token. + Token FastMathTok; + FastMathTok.startToken(); + FastMathTok.setKind(tok::annot_pragma_fast_math); + FastMathTok.setLocation(PragmaName.getLocation()); + FastMathTok.setAnnotationEndLoc(PragmaName.getLocation()); + FastMathTok.setAnnotationValue(reinterpret_cast(AnnotValue)); + TokenList.push_back(FastMathTok); + } + + if (Tok.isNot(tok::eod)) { + PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) + << "clang fast_math"; + return; + } + + auto TokenArray = llvm::make_unique(TokenList.size()); + std::copy(TokenList.begin(), TokenList.end(), TokenArray.get()); + + PP.EnterTokenStream(std::move(TokenArray), TokenList.size(), + /*DisableMacroExpansion=*/false); +} + +void Parser::HandlePragmaFastMath() { + assert(Tok.is(tok::annot_pragma_fast_math)); + auto *AnnotValue = + reinterpret_cast(Tok.getAnnotationValue()); + + Actions.ActOnPragmaFPContract(AnnotValue->FlagValue ? LangOptions::FPC_Fast + : LangOptions::FPC_Off); + ConsumeToken(); // The annotation token. +} + /// \brief Parses loop or unroll pragma hint value and fills in Info. static bool ParseLoopHintValue(Preprocessor &PP, Token &Tok, Token PragmaName, Token Option, bool ValueInParens, Index: lib/Parse/ParseStmt.cpp =================================================================== --- lib/Parse/ParseStmt.cpp +++ lib/Parse/ParseStmt.cpp @@ -341,6 +341,12 @@ ConsumeToken(); return StmtError(); + case tok::annot_pragma_fast_math: + ProhibitAttributes(Attrs); + Diag(Tok, diag::err_pragma_fast_math_scope); + ConsumeToken(); + return StmtError(); + case tok::annot_pragma_opencl_extension: ProhibitAttributes(Attrs); HandlePragmaOpenCLExtension(); @@ -900,6 +906,9 @@ case tok::annot_pragma_fp_contract: HandlePragmaFPContract(); break; + case tok::annot_pragma_fast_math: + HandlePragmaFastMath(); + break; case tok::annot_pragma_ms_pointers_to_members: HandlePragmaMSPointersToMembers(); break; Index: lib/Parse/Parser.cpp =================================================================== --- lib/Parse/Parser.cpp +++ lib/Parse/Parser.cpp @@ -690,6 +690,9 @@ case tok::annot_pragma_fp_contract: HandlePragmaFPContract(); return nullptr; + case tok::annot_pragma_fast_math: + HandlePragmaFastMath(); + break; case tok::annot_pragma_opencl_extension: HandlePragmaOpenCLExtension(); return nullptr; Index: lib/Sema/SemaAttr.cpp =================================================================== --- lib/Sema/SemaAttr.cpp +++ lib/Sema/SemaAttr.cpp @@ -447,19 +447,16 @@ } } -void Sema::ActOnPragmaFPContract(tok::OnOffSwitch OOS) { - switch (OOS) { - case tok::OOS_ON: +void Sema::ActOnPragmaFPContract(LangOptions::FPContractModeKind FPC) { + switch (FPC) { + case LangOptions::FPC_On: FPFeatures.setAllowFPContractWithinStatement(); break; - case tok::OOS_OFF: - FPFeatures.setDisallowFPContract(); + case LangOptions::FPC_Fast: + FPFeatures.setAllowFPContractAcrossStatement(); break; - case tok::OOS_DEFAULT: - if (getLangOpts().getDefaultFPContractMode() == LangOptions::FPC_On) - FPFeatures.setAllowFPContractWithinStatement(); - else - FPFeatures.setDisallowFPContract(); + case LangOptions::FPC_Off: + FPFeatures.setDisallowFPContract(); break; } } Index: test/CodeGen/fp-contract-fast-pragma.cpp =================================================================== --- /dev/null +++ test/CodeGen/fp-contract-fast-pragma.cpp @@ -0,0 +1,69 @@ +// RUN: %clang_cc1 -O3 -triple %itanium_abi_triple -emit-llvm -o - %s | FileCheck %s + +// Is FP_CONTRACT honored in a simple case? +float fp_contract_1(float a, float b, float c) { +// CHECK: _Z13fp_contract_1fff +// CHECK: %[[M:.+]] = fmul contract float %a, %b +// CHECK-NEXT: fadd contract float %[[M]], %c +#pragma clang fast_math contract_fast(on) + return a * b + c; +} + +// Is FP_CONTRACT state cleared on exiting compound statements? +float fp_contract_2(float a, float b, float c) { + // CHECK: _Z13fp_contract_2fff + // CHECK: %[[M:.+]] = fmul float %a, %b + // CHECK-NEXT: fadd float %[[M]], %c + { +#pragma clang fast_math contract_fast(on) + } + return a * b + c; +} + +// Does FP_CONTRACT survive template instantiation? +class Foo {}; +Foo operator+(Foo, Foo); + +template +T template_muladd(T a, T b, T c) { +#pragma clang fast_math contract_fast(on) + return a * b + c; +} + +float fp_contract_3(float a, float b, float c) { + // CHECK: _Z13fp_contract_3fff + // CHECK: %[[M:.+]] = fmul contract float %a, %b + // CHECK-NEXT: fadd contract float %[[M]], %c + return template_muladd(a, b, c); +} + +template +class fp_contract_4 { + float method(float a, float b, float c) { +#pragma clang fast_math contract_fast(on) + return a * b + c; + } +}; + +template class fp_contract_4; +// CHECK: _ZN13fp_contract_4IiE6methodEfff +// CHECK: %[[M:.+]] = fmul contract float %a, %b +// CHECK-NEXT: fadd contract float %[[M]], %c + +// Check file-scoped FP_CONTRACT +#pragma clang fast_math contract_fast(on) +float fp_contract_5(float a, float b, float c) { + // CHECK: _Z13fp_contract_5fff + // CHECK: %[[M:.+]] = fmul contract float %a, %b + // CHECK-NEXT: fadd contract float %[[M]], %c + return a * b + c; +} + +// Verify that we can handle multiple flags on the same pragma +#pragma clang fast_math contract_fast(on) contract_fast(off) +float fp_contract_6(float a, float b, float c) { + // CHECK: _Z13fp_contract_6fff + // CHECK: %[[M:.+]] = fmul float %a, %b + // CHECK-NEXT: fadd float %[[M]], %c + return a * b + c; +} Index: test/Parser/cxx11-stmt-attributes.cpp =================================================================== --- test/Parser/cxx11-stmt-attributes.cpp +++ test/Parser/cxx11-stmt-attributes.cpp @@ -80,5 +80,6 @@ { [[ ]] // expected-error {{an attribute list cannot appear here}} #pragma STDC FP_CONTRACT ON // expected-error {{can only appear at file scope or at the start of a compound statement}} +#pragma clang fast_math contract_fast(on) // expected-error {{can only appear at file scope or at the start of a compound statement}} } } Index: test/Parser/pragma-fast-math.cpp =================================================================== --- /dev/null +++ test/Parser/pragma-fast-math.cpp @@ -0,0 +1,64 @@ +// RUN: %clang_cc1 -std=c++11 -verify %s + +void test_0(int *List, int Length) { +/* expected-error@+1 {{missing option; expected contract_fast}} */ +#pragma clang fast_math + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} +void test_1(int *List, int Length) { +/* expected-error@+1 {{invalid option 'blah'; expected contract_fast}} */ +#pragma clang fast_math blah + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} + +void test_3(int *List, int Length) { +/* expected-error@+1 {{expected '('}} */ +#pragma clang fast_math contract_fast on + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} + +void test_4(int *List, int Length) { +/* expected-error@+1 {{unexpected argument 'while' to '#pragma clang fast_math contract_fast'; expected 'on' or 'off'}} */ +#pragma clang fast_math contract_fast(while) + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} + +void test_5(int *List, int Length) { +/* expected-error@+1 {{unexpected argument 'maybe' to '#pragma clang fast_math contract_fast'; expected 'on' or 'off'}} */ +#pragma clang fast_math contract_fast(maybe) + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} + +void test_6(int *List, int Length) { +/* expected-error@+1 {{expected ')'}} */ +#pragma clang fast_math contract_fast(on + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} + +void test_7(int *List, int Length) { +/* expected-warning@+1 {{extra tokens at end of '#pragma clang fast_math' - ignored}} */ +#pragma clang fast_math contract_fast(on) * + for (int i = 0; i < Length; i++) { + List[i] = i; + } +} + +void test_8(int *List, int Length) { + for (int i = 0; i < Length; i++) { + List[i] = i; +/* expected-error@+1 {{'#pragma clang fast_math' can only appear at file scope or at the start of a compound statement}} */ +#pragma clang fast_math contract_fast(on) + } +}