[Fortran] Re-enable FCVS FM905 & FM907

[rovka]

Summary

FM905 and FM907 have been disabled because they use list-directed
output, which gives the compiler some freedom regarding the exact format
and precision of the output. Our reference output had been obtained with
gfortran, and flang's output did not match.

This patch re-enables the tests, but switches the reference outputs to
match what flang produces. By default, we will check the output exactly.
However, the patch also introduces a new CMake variable,
FCVS_ALLOW_FLEXIBLE_OUTPUT, which can be set when running the
test-suite with other compilers (or even with other flags or environment
variables) and which will check the output more loosely. In this patch,
this only affect whether or not we ignore whitespace differences in the
output.

Note that this patch is not enough for the test-suite to pass with
gfortran, even when FCVS_ALLOW_FLEXIBLE_OUTPUT is enabled. Future
patches will try to fix that on a best-effort basis. Other fixes may be
required for other compilers or other compiler options.

[awarzynski]

LGTM, thank you for doing this!

You probably want to update the following as well:

• https://github.com/llvm/llvm-test-suite/blob/25c31a4e8b54b5e2d4e03b21aa6631fdb248dc6e/Fortran/UnitTests/fcvs21_f95/CMakeLists.txt#L4
• https://github.com/llvm/llvm-test-suite/blob/25c31a4e8b54b5e2d4e03b21aa6631fdb248dc6e/Fortran/UnitTests/fcvs21_f95/README#L49

[rovka]

In D131754#3718904, @awarzynski wrote:

LGTM, thank you for doing this!

You probably want to update the following as well:

• https://github.com/llvm/llvm-test-suite/blob/25c31a4e8b54b5e2d4e03b21aa6631fdb248dc6e/Fortran/UnitTests/fcvs21_f95/CMakeLists.txt#L4
• https://github.com/llvm/llvm-test-suite/blob/25c31a4e8b54b5e2d4e03b21aa6631fdb248dc6e/Fortran/UnitTests/fcvs21_f95/README#L49

Good catch, thanks! I'll update the CMakeLists.txt. I don't think the README needs to be updated, since the changes only affect the reference outputs, which weren't part of the original sources.

[DavidSpickett]

These are failing on https://lab.llvm.org/buildbot/#/builders/179/builds/4259, if you didn't already know.

Not sure why it took ~4 days to show up though.

[rovka]

Sorry, this is because the buildbot is still running the test-suite with
flang-to-external-fc instead of flang-new. It seems only the staging
buildmaster got restarted to include the change to flang-new :(

Fortran/UnitTests/fcvs21_f95/CMakeLists.txt

 # NIST Fortran 77 Test Suite
 # http://www.fortran-2000.com/ArnaudRecipes/fcvs21_f95.html
 #
 # Reference results generated with gfortran -v:
 # Using built-in specs.
 # COLLECT_GCC=gfortran
 # COLLECT_LTO_WRAPPER=/usr/lib/gcc/x86_64-linux-gnu/7/lto-wrapper
 # OFFLOAD_TARGET_NAMES=nvptx-none
 # OFFLOAD_TARGET_DEFAULT=1
 # Target: x86_64-linux-gnu
 # Configured with: ../src/configure -v --with-pkgversion='Ubuntu 7.5.0-3ubuntu1~18.04' --with-bugurl=file:///usr/share/doc/gcc-7/README.Bugs --enable-languages=c,ada,c++,go,brig,d,fortran,objc,obj-c++ --prefix=/usr --with-gcc-major-version-only --program-suffix=-7 --program-prefix=x86_64-linux-gnu- --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --libdir=/usr/lib --enable-nls --enable-bootstrap --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --with-default-libstdcxx-abi=new --enable-gnu-unique-object --disable-vtable-verify --enable-libmpx --enable-plugin --enable-default-pie --with-system-zlib --with-target-system-zlib --enable-objc-gc=auto --enable-multiarch --disable-werror --with-arch-32=i686 --with-abi=m64 --with-multilib-list=m32,m64,mx32 --enable-multilib --with-tune=generic --enable-offload-targets=nvptx-none --without-cuda-driver --enable-checking=release --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu
 # Thread model: posix
 # gcc version 7.5.0 (Ubuntu 7.5.0-3ubuntu1~18.04)
 #
+# Reference results for tests 905 and 907 generated with flang ~15.0.0 on
+# Ubuntu 20.04 (aarch64-unknown-linux-gnu).
+#
 # See additional comments in `driver_run` script should reference
 # results ever need to be updated.
 #
 # Following test have been intentionally ommitted due to feature
 # deletion in Fortran 95 standard:
 # F95: 103, 111, 252, 253, 255, 257, 259, 260, 719
 #
 # We omit these tests because they never pass with any recent
 # compiler:
 # 001, 406, 923
 # 
 # Tests make use of arithmetic IF which is now a deleted feature
 # in Fortran 2008
 # 
 # Note that the NIST license allows redistribution but requires
 # explicit acknowledgement of what has been changed. Please
 # amend README file if you modify the tests.
 
 # clean-up fort.* files otherwise one of the tests will fail
 llvm_test_prepare(rm -f %S/fort.*)
 
+# Different compilers or different compiler options may produce slightly
+# different but still valid outputs for some of these tests. By default, we will
+# check that the outputs match the reference outputs exactly. However, when this
+# option is set, we will check less strictly.
+option(FCVS_ALLOW_FLEXIBLE_OUTPUT "Don't match the output of FCVS tests exactly")
+
 # Group 1:
 # Tests 905 and 907 use list-directed output, for which the standard allows some
 # flexibility, e.g., with regards to how many white space characters or
 # floating point decimals to print. Treat them separately.
-# FIXME: Reenable these tests after moving the buildbots from using
-# flang-to-external-fc to using flang-new.
 set(SPECIAL_CASES "FM905.f" "FM907.f")
-if (NOT CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
-  set(Source ${SPECIAL_CASES})
+if (FCVS_ALLOW_FLEXIBLE_OUTPUT)
   set(FP_IGNOREWHITESPACE ON)
-  llvm_singlesource()
-  set(Source)
-endif()
+endif()
 
 # Group 2:
 # Generic case. Make sure to exclude the special cases which have already been
 # added as part of Group 1.
 file(GLOB Source CONFIGURE_DEPENDS *.f)
 foreach (SPECIAL_CASE ${SPECIAL_CASES})
   list(FILTER Source EXCLUDE REGEX ${SPECIAL_CASE})
 endforeach()

Fortran/UnitTests/fcvs21_f95/FM905.reference_output

-------------------------------------------------------------------------------
                                                  AND THE NUMBER OF DECIMAL      
                                                  PLACES FOR REAL NUMBERS ARE    
                                                  PROCESSOR DEPENDENT.           
                                                  EITHER E OR F FORMAT MAY BE    
                                                  USED FOR REAL NUMBERS.         
 
      1    INSPECT
                  COMPUTED= 
-           2
                  CORRECT=  
        2
      2    INSPECT
                  COMPUTED= 
-           1           3           5           7           9
+ 1 3 5 7 9
                  CORRECT=  
        1  3  5  7  9
      3    INSPECT
                  COMPUTED= 
-   2.50000000    
+ 2.5
                  CORRECT=  
        2.5
      4    INSPECT
                  COMPUTED= 
-   2.50000003E-11  0.250000000       250.000000       2.50000000E+09
+ 2.5E-11 .25 250. 2.5E+09
                  CORRECT=  
         2.5E-11  0.25  250.0   2.5E+09
      5    INSPECT
                  COMPUTED= 
  T F T
                  CORRECT=  
        T  F  T
      6    INSPECT
                  COMPUTED= 
  ONE  TWO  THREEFOUR 
                  CORRECT=  
        ONE  TWO  THREEFOUR 
      7    INSPECT
                  COMPUTED= 
-          -3   15.2500000     HELLO T
+ -3 15.25 HELLO T
                  CORRECT=  
        -3  15.25  HELLO  T
      8    INSPECT
                  COMPUTED= 
  5 O'CLOCK
                  CORRECT=  
        5 O'CLOCK
      9    INSPECT
                  COMPUTED= 
- SHORTTHIS IS A LONGER CHARACTER STRING123456789012345678901234567890123456789012345678901234567890123456789012          
+ SHORTTHIS IS A LONGER CHARACTER STRING1234567890123456789012345678901234567890
+ 12345678901234567890123456789012          
                  CORRECT=  
  SHORT  THIS IS A LONGER CHARACTER STRING 123456789012345678901234567890123456789
  012345678901234567890123456789012
     10    INSPECT
                  COMPUTED= 
-           5           5           5           5           5
+ 5 5 5 5 5
                  CORRECT=  
        5  5  5  5  5  OR  5*5
  
  -------------------------------------------------------------------------------
  
                           0 TESTS PASSED
                           0 TESTS FAILED
                           0 TESTS DELETED
                          10 TESTS REQUIRE INSPECTION
                          10 OF  10 TESTS EXECUTED
  
  *FM905END*              END OF TEST - FM905
 
  *NO PROJECT*              *NO COMPANY NAME*      *   *NO TAPE* /*NO TAPE DATE
  FOR OFFICIAL USE ONLY                                        COPYRIGHT  1982
 exit 0

Fortran/UnitTests/fcvs21_f95/FM907.reference_output

-------------------------------------------------------------------------------
                                                  OR COMPLEX NUMBERS ARE         
                                                  PROCESSOR DEPENDENT.           
                                                  EITHER E OR F FORMAT MAY BE    
                                                  USED FOR DOUBLE PRECISION OR   
                                                  COMPLEX NUMBERS.               
 
      1    INSPECT
                  COMPUTED= 
-   2.5000000000000000     
+ 2.5
                  CORRECT=  
        2.5
      2    INSPECT
                  COMPUTED= 
-             (3.00000000,4.00000000)
+ (3.,4.)
                  CORRECT=  
         (3.0,4.0)
      3    INSPECT
                  COMPUTED= 
-   2.5000000000000000        2.5000000000000002E-010   25000000000.000000     
+ 2.5 2.5E-10 25000000000.
                  CORRECT=  
        2.5  2.5D-10  2.5D+10
      4    INSPECT
                  COMPUTED= 
-             (0.00000000,1.00000000)             (8.00000000,10.0000000)            (-5.00000000,0.00000000)             (0.00000000,0.00000000)
+ (0.,1.) (8.,10.) (-5.,0.) (0.,0.)
                  CORRECT=  
         (0.0,1.0)   (8.0,10.0)   (-5.0,0.0)   (0.0,0.0)
      5    INSPECT
                  COMPUTED= 
-             (3.00000000,4.00000000)   5.0000000000000000       -5.0000000000000000                (-3.00000000,-4.00000000)
+ (3.,4.) 5. -5. (-3.,-4.)
                  CORRECT=  
         (3.0,4.0)  5.0  -5.0   (-3.0,-4.0)
      6    INSPECT
                  COMPUTED= 
-             (6.00000000,9.00000000)
+ (6.,9.)
                  CORRECT=  
         (6.0,9.0)
      7    INSPECT
                  COMPUTED= 
-   3.25000000    
+ 3.25
                  CORRECT=  
        3.25
      8    INSPECT
                  COMPUTED= 
-            (2.00000000,-3.00000000) T   15.6250000     GOODBYE FOR NOW
+ (2.,-3.) T 15.625 GOODBYE FOR NOW
                  CORRECT=  
         (2.0,-3.0)  T  15.625  GOODBYE  FOR NOW
  
  -------------------------------------------------------------------------------
  
                           0 TESTS PASSED
                           0 TESTS FAILED
                           0 TESTS DELETED
                           8 TESTS REQUIRE INSPECTION
                           8 OF   8 TESTS EXECUTED
  
  *FM907END*              END OF TEST - FM907
 
  *NO PROJECT*              *NO COMPANY NAME*      *   *NO TAPE* /*NO TAPE DATE
  FOR OFFICIAL USE ONLY                                        COPYRIGHT  1982
 exit 0