[flang] Allow compiler directives between OpenACC constructs and loops

[unterumarmung]

Summary

This change allows to place arbitrary compiler directives between
OpenACCLoopConstruct/OpenACCCombinedConstruct and DO loops.
This is implemented by moving compiler directives before the
corresponding OpenACC construct.

More info can be found here: https://discourse.llvm.org/t/compiler-directives-between-openacc-constructs-and-do-loops

flang/lib/Semantics/canonicalize-acc.cpp

 //===-- lib/Semantics/canonicalize-acc.cpp --------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "canonicalize-acc.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/ADT/SmallPtrSet.h"
 
 // After Loop Canonicalization, rewrite OpenACC parse tree to make OpenACC
 // Constructs more structured which provide explicit scopes for later
 // structural checks and semantic analysis.
 //   1. move structured DoConstruct into
 //      OpenACCLoopConstruct. Compilation will not proceed in case of errors
 //      after this pass.
 //   2. move structured DoConstruct into OpenACCCombinedConstruct. Move
 //      AccEndCombinedConstruct into OpenACCCombinedConstruct if present.
 //      Compilation will not proceed in case of errors after this pass.
 namespace Fortran::semantics {
 
 using namespace parser::literals;
 
+template <typename T> static bool IsCompilerDirective(const T &construct) {
+  return parser::Unwrap<parser::CompilerDirective>(construct) != nullptr;
+}
+
 class CanonicalizationOfAcc {
 public:
   template <typename T> bool Pre(T &) { return true; }
   template <typename T> void Post(T &) {}
   CanonicalizationOfAcc(parser::Messages &messages) : messages_{messages} {}
 
   void Post(parser::Block &block) {
-    for (auto it{block.begin()}; it != block.end(); ++it) {
-      if (auto *accLoop{parser::Unwrap<parser::OpenACCLoopConstruct>(*it)}) {
+    using CommonType = void *;
+    llvm::SmallPtrSet<CommonType, 16> seen;
+    for (auto it{block.begin()}; it != block.end(); it++) {
+      if (auto *accLoop{parser::Unwrap<parser::OpenACCLoopConstruct>(*it)};
+          accLoop && !seen.contains(accLoop)) {
         RewriteOpenACCLoopConstruct(*accLoop, block, it);
+        seen.insert(accLoop);
       } else if (auto *accCombined{
-                     parser::Unwrap<parser::OpenACCCombinedConstruct>(*it)}) {
+                     parser::Unwrap<parser::OpenACCCombinedConstruct>(*it)};
+                 accCombined && !seen.contains(accCombined)) {
         RewriteOpenACCCombinedConstruct(*accCombined, block, it);
+        seen.insert(accCombined);
       } else if (auto *endDir{
                      parser::Unwrap<parser::AccEndCombinedDirective>(*it)}) {
         // Unmatched AccEndCombinedDirective
         messages_.Say(endDir->v.source,
             "The %s directive must follow the DO loop associated with the "
             "loop construct"_err_en_US,
             parser::ToUpperCaseLetters(endDir->v.source.ToString()));
       }
for (const auto &clause : accClauseList.v) {
       if (const auto *tileClause =
               std::get_if<parser::AccClause::Tile>(&clause.u)) {
         const parser::AccTileExprList &tileExprList = tileClause->v;
         const std::list<parser::AccTileExpr> &listTileExpr = tileExprList.v;
         std::size_t tileArgNb = listTileExpr.size();
 
         const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
         if (outer->IsDoConcurrent()) {
-          return; // Tile is not allowed on DO CONURRENT
+          return; // Tile is not allowed on DO CONCURRENT
         }
         for (const parser::DoConstruct *loop{&*outer}; loop && tileArgNb > 0;
              --tileArgNb) {
           const auto &block{std::get<parser::Block>(loop->t)};
           const auto it{block.begin()};
           loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
                                    : nullptr;
         }
private:
 
   void RewriteOpenACCLoopConstruct(parser::OpenACCLoopConstruct &x,
       parser::Block &block, parser::Block::iterator it) {
     // Check the sequence of DoConstruct in the same iteration
     //
     // Original:
     //   ExecutableConstruct -> OpenACCConstruct -> OpenACCLoopConstruct
     //     ACCBeginLoopDirective
+    //   [CompilerDirective ...]
     //   ExecutableConstruct -> DoConstruct
     //
     // After rewriting:
+    //   [CompilerDirective ...]
     //   ExecutableConstruct -> OpenACCConstruct -> OpenACCLoopConstruct
     //     AccBeginLoopDirective
     //     DoConstruct
-    parser::Block::iterator nextIt;
     auto &beginDir{std::get<parser::AccBeginLoopDirective>(x.t)};
     auto &dir{std::get<parser::AccLoopDirective>(beginDir.t)};
 
-    nextIt = it;
-    if (++nextIt != block.end()) {
+    auto nextIt{it};
+    while (++nextIt != block.end() && IsCompilerDirective(*nextIt)) {
+      std::swap(*std::prev(nextIt), *nextIt);
+    }
+
+    if (nextIt != block.end()) {
       if (auto *doCons{parser::Unwrap<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
           std::get<std::optional<parser::DoConstruct>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
         } else {
           messages_.Say(dir.source,
private:
 
   void RewriteOpenACCCombinedConstruct(parser::OpenACCCombinedConstruct &x,
       parser::Block &block, parser::Block::iterator it) {
     // Check the sequence of DoConstruct in the same iteration
     //
     // Original:
     //   ExecutableConstruct -> OpenACCConstruct -> OpenACCCombinedConstruct
     //     ACCBeginCombinedDirective
+    //   [CompilerDirective ...]
     //   ExecutableConstruct -> DoConstruct
     //   ExecutableConstruct -> AccEndCombinedDirective (if available)
     //
     // After rewriting:
+    //   [CompilerDirective ...]
     //   ExecutableConstruct -> OpenACCConstruct -> OpenACCCombinedConstruct
     //     ACCBeginCombinedDirective
     //     DoConstruct
     //     AccEndCombinedDirective (if available)
-    parser::Block::iterator nextIt;
+
     auto &beginDir{std::get<parser::AccBeginCombinedDirective>(x.t)};
     auto &dir{std::get<parser::AccCombinedDirective>(beginDir.t)};
 
-    nextIt = it;
-    if (++nextIt != block.end()) {
+    auto nextIt{it};
+
+    while (++nextIt != block.end() && IsCompilerDirective(*nextIt)) {
+      std::swap(*std::prev(nextIt), *nextIt);
+    }
+
+    if (nextIt != block.end()) {
       if (auto *doCons{parser::Unwrap<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
           std::get<std::optional<parser::DoConstruct>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match AccEndCombinedDirective
           if (nextIt != block.end()) {

flang/test/Semantics/OpenACC/acc-move-compiler-directives.f90

+! RUN: %flang_fc1 -fopenacc -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s --check-prefix=AFTER
+! RUN: %flang_fc1 -fopenacc -fdebug-dump-parse-tree-no-sema %s 2>&1 | FileCheck %s --check-prefix=BEFORE
+subroutine foo
+  integer :: bar
+
+  bar = 1
+
+! BEFORE:      | | ExecutionPartConstruct -> ExecutableConstruct -> OpenACCConstruct -> OpenACCLoopConstruct
+! BEFORE-NEXT: | | | AccBeginLoopDirective
+! BEFORE-NEXT: | | | | AccLoopDirective -> llvm::acc::Directive = loop
+! BEFORE-NEXT: | | | | AccClauseList -> 
+! BEFORE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> CompilerDirective -> NameValue
+! BEFORE-NEXT: | | | Name = 'ivdep'
+
+! AFTER:      | | ExecutionPartConstruct -> ExecutableConstruct -> CompilerDirective -> NameValue
+! AFTER-NEXT: | | | Name = 'ivdep'
+! AFTER-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenACCConstruct -> OpenACCLoopConstruct
+! AFTER-NEXT: | | | AccBeginLoopDirective
+
+  !$acc loop
+  !dir$ ivdep
+  do i = 1, 100, 1
+    bar = bar + i
+  end do
+
+! BEFORE:      | | ExecutionPartConstruct -> ExecutableConstruct -> OpenACCConstruct -> OpenACCCombinedConstruct
+! BEFORE-NEXT: | | | AccBeginCombinedDirective
+! BEFORE-NEXT: | | | | AccCombinedDirective -> llvm::acc::Directive = parallel loop
+! BEFORE-NEXT: | | | | AccClauseList -> 
+! BEFORE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> CompilerDirective -> NameValue
+! BEFORE-NEXT: | | | Name = 'ivdep'
+
+! AFTER:      | | ExecutionPartConstruct -> ExecutableConstruct -> CompilerDirective -> NameValue
+! AFTER-NEXT: | | | Name = 'ivdep'
+! AFTER-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenACCConstruct -> OpenACCCombinedConstruct
+! AFTER-NEXT: | | | AccBeginCombinedDirective
+
+  !$acc parallel loop
+  !dir$ ivdep
+  do i = 1, 100, 1
+    bar = bar + i
+  end do
+end subroutine foo
 No newline at end of file