On Tue, 18 Mar 2025, Jakub Jelinek wrote:
> Hi!
>
> The COBOL tests has many tests which just dump emit lots of output
> to stdout and want to compare it against expected output.
> We have the dg-output directive, but if one needs more than dozens
> of lines in the output, adding hundreds of dg-output directives to
> each source uses too much memory and is harder to maintain.
>
> The following patch offers an alternative, dg-output-file
> directive where one can supply a text file with expected output
> (no regexp matching in that case, just exact output, except that it
> handles different line ending styles (for the expected file
> using tcl gets, for the actual output skips over \n, \r\n or \r).
> And a newline at the end of the whole output is optional (in the actual
> output, because I think some boards get it eaten).
>
> Tested on x86_64-linux and i686-linux, ok for trunk?
LGTM.
Thanks,
Richard.
> Also tested with addition or subtraction of some characters from the
> expected output files and saw FAILs with appropriate messages.
>
> 2025-03-18 Jakub Jelinek <ja...@redhat.com>
>
> * doc/sourcebuild.texi (dg-output-file): Document.
>
> * lib/gcc-dg.exp (${tool}-load): If output-file is set, compare
> combined output against content of the [lindex ${output-file} 1]
> file.
> (dg-output-file): New directive.
> * lib/dg-test-cleanup.exp (cleanup-after-saved-dg-test): Clear
> output-file variable.
> * gcc.dg/dg-output-file-1.c: New test.
> * gcc.dg/dg-output-file-1-lp64.txt: New test.
> * gcc.dg/dg-output-file-1-ilp32.txt: New test.
>
> --- gcc/doc/sourcebuild.texi.jj 2025-03-11 09:18:21.750133577 +0100
> +++ gcc/doc/sourcebuild.texi 2025-03-18 14:41:59.253345259 +0100
> @@ -1315,6 +1315,10 @@ Prune messages matching @var{regexp} fro
> @item @{ dg-output @var{regexp} [@{ target/xfail @var{selector} @}] @}
> This DejaGnu directive compares @var{regexp} to the combined output
> that the test executable writes to @file{stdout} and @file{stderr}.
> +
> +@item @{ dg-output-file @var{file} [@{ target/xfail @var{selector} @}] @}
> +Compares the content of @var{file} against the combined output that the
> +test executable writes to @file{stdout} and @file{stderr}.
> @end table
>
> @subsubsection Specify environment variables for a test
> --- gcc/testsuite/lib/gcc-dg.exp.jj 2025-03-13 14:05:09.707017142 +0100
> +++ gcc/testsuite/lib/gcc-dg.exp 2025-03-18 14:37:32.476088575 +0100
> @@ -473,6 +473,7 @@ if { [info procs ${tool}_load] != [list]
> global tool
> global shouldfail
> global set_target_env_var
> + global output-file
>
> set saved_target_env_var [list]
> if { [info exists set_target_env_var] \
> @@ -497,6 +498,75 @@ if { [info procs ${tool}_load] != [list]
> }
>
> set result [list [lindex $result 0] [prune_file_path [lindex $result
> 1]]]
> + if { [info exists output-file] && [lindex $result 0] eq "pass" } {
> + if { [lindex ${output-file} 0] eq "F" } {
> + setup_xfail "*-*-*"
> + }
> + set output [lindex $result 1]
> + set idx 0
> + set linenum 1
> + set outfile [open [lindex ${output-file} 1]]
> + set do_fail 0
> + set name [file tail [lindex ${output-file} 1]]
> + verbose "output-file args is $args program is $program" 1
> + while { [gets $outfile line] >= 0 } {
> + if { $linenum != 1 } {
> + set c [string index $output $idx]
> + if { $c eq "\n" } {
> + set idx [expr $idx + 1]
> + } elseif { $c eq "\r" } {
> + set idx [expr $idx + 1]
> + set c [string index $output $idx]
> + if { $c eq "\n" } {
> + set idx [expr $idx + 1]
> + }
> + } else {
> + set do_fail 1
> + fail "$name output file test"
> + send_log "Unexpected character $c on line [expr
> $linenum - 1] where new-line expected\n"
> + verbose "Failed test for output line [expr $linenum -
> 1]" 3
> + break
> + }
> + }
> + set len [string length $line]
> + set output_line [string range $output $idx [expr $idx + $len -
> 1]]
> + if { $line ne $output_line } {
> + set do_fail 1
> + fail "$name output file test"
> + send_log "Output line $linenum was:\n$output_line\nShould
> match (from [lindex ${output-file} 1]):\n$line\n"
> + verbose "Failed test for output line $linenum $line" 3
> + break
> + }
> + set idx [expr $idx + $len]
> + incr linenum
> + }
> + close $outfile
> + if { $do_fail == 0 } {
> + set c [string index $output $idx]
> + if { $c eq "\n" } {
> + set idx [expr $idx + 1]
> + } elseif { $c eq "\r" } {
> + set idx [expr $idx + 1]
> + set c [string index $output $idx]
> + if { $c eq "\n" } {
> + set idx [expr $idx + 1]
> + }
> + } else {
> + incr linenum -1
> + }
> + set c [string index $output $idx]
> + if { $c ne "" } {
> + fail "$name output file test"
> + send_log "Unexpected character $c on line $linenum where
> and of output expected\n"
> + verbose "Failed test for output line $linenum" 3
> + break
> + } else {
> + pass "$name output file test"
> + verbose "Passed test for output file [lindex ${output-file}
> 1]" 3
> + }
> + }
> + unset output
> + }
> return $result
> }
> }
> @@ -575,6 +645,35 @@ proc restore-compiler-env-var { } {
> }
> }
>
> +# Indicate expected program output in a file.
> +#
> +proc dg-output-file { args } {
> + global output-file
> + global srcdir subdir
> +
> + if { [llength $args] > 3 } {
> + error "[lindex $args 0]: too many arguments"
> + }
> +
> + # Allow target dependent output.
> +
> + if { ![info exists output-file] } {
> + set output-file "P"
> + }
> + set expected [lindex ${output-file} 0]
> + if { [llength $args] >= 3 } {
> + switch -- [dg-process-target [lindex $args 2]] {
> + "N" { return }
> + "S" { }
> + "F" { set expected "F" }
> + # Don't override a previous xfail.
> + "P" { }
> + }
> + }
> +
> + set output-file [list $expected $srcdir/$subdir/[lindex $args 1]]
> +}
> +
> # Utility routines.
>
> #
> --- gcc/testsuite/lib/dg-test-cleanup.exp.jj 2025-01-02 11:47:41.684062215
> +0100
> +++ gcc/testsuite/lib/dg-test-cleanup.exp 2025-03-18 12:05:55.714299937
> +0100
> @@ -45,6 +45,7 @@ if { [info procs saved-dg-test] == [list
> global multiline_expected_outputs
> global freeform_regexps
> global save_linenr_varnames
> + global output-file
>
> set additional_files ""
> set additional_sources ""
> @@ -70,6 +71,9 @@ if { [info procs saved-dg-test] == [list
> if [info exists testname_with_flags] {
> unset testname_with_flags
> }
> + if [info exists output-file] {
> + unset output-file
> + }
> set nn_line_numbers_enabled 0
> set multiline_expected_outputs []
> set freeform_regexps []
> --- gcc/testsuite/gcc.dg/dg-output-file-1.c.jj 2025-03-18
> 12:30:38.355563473 +0100
> +++ gcc/testsuite/gcc.dg/dg-output-file-1.c 2025-03-18 12:48:26.306636698
> +0100
> @@ -0,0 +1,13 @@
> +/* { dg-do run { target { lp64 || ilp32 } } } */
> +/* { dg-options "-O2" } */
> +/* { dg-output-file "dg-output-file-1-lp64.txt" { target lp64 } } */
> +/* { dg-output-file "dg-output-file-1-ilp32.txt" { target ilp32 } } */
> +
> +int
> +main ()
> +{
> + __builtin_printf ("This is a test output for %s target\n"
> + "to verify\n"
> + "dg-output-file directive\n",
> + __SIZEOF_LONG__ * __CHAR_BIT__ == 64 ? "lp64" : "ilp32");
> +}
> --- gcc/testsuite/gcc.dg/dg-output-file-1-lp64.txt.jj 2025-03-18
> 12:31:08.514141366 +0100
> +++ gcc/testsuite/gcc.dg/dg-output-file-1-lp64.txt 2025-03-18
> 12:32:12.080251677 +0100
> @@ -0,0 +1,3 @@
> +This is a test output for lp64 target
> +to verify
> +dg-output-file directive
> --- gcc/testsuite/gcc.dg/dg-output-file-1-ilp32.txt.jj 2025-03-18
> 12:31:38.841716894 +0100
> +++ gcc/testsuite/gcc.dg/dg-output-file-1-ilp32.txt 2025-03-18
> 14:37:50.289838618 +0100
> @@ -0,0 +1,3 @@
> +This is a test output for ilp32 target
> +to verify
> +dg-output-file directive
>
> Jakub
>
>
--
Richard Biener <rguent...@suse.de>
SUSE Software Solutions Germany GmbH,
Frankenstrasse 146, 90461 Nuernberg, Germany;
GF: Ivo Totev, Andrew McDonald, Werner Knoblich; (HRB 36809, AG Nuernberg)
