path: root/utils/runstdtest/runstdtest.prl

#
# The perl script requires the following variables to be bound
# to something meaningful before it will operate correctly:
#   
#   DEFAULT_TMPDIR
#   CONTEXT_DIFF
#   RM
#
# Given:
#	* a program to run (1st arg)
# 	* some "command-line opts" ( -O<opt1> -O<opt2> ... )
#	    [default: anything on the cmd line this script doesn't recognise ]
#	  the first opt not starting w/ "-" is taken to be an input
#	  file and (if it exists) is grepped for "what's going on here"
#	  comments (^-- !!!).
#	* a file to feed to stdin ( -i<file> ) [default: /dev/null ]
#	* a "time" command to use (-t <cmd>).
#
#	* alternatively, a "-script <script>" argument says: run the
#   	  named Bourne-shell script to do the test.  It's passed the
#	  pgm-to-run as the one-and-only arg.
#
# Run the program with those options and that input, and check:
# if we get...
# 
# 	* an expected exit status ( -x <val> ) [ default 0 ]
# 	* expected output on stdout ( -o1 <file> ) [ default /dev/null ]
#		( we'll accept one of several...)
# 	* expected output on stderr ( -o2 <file> ) [ default /dev/null ]
#		( we'll accept one of several...)
#
#	(if the expected-output files' names end in .Z, then
#	 they are uncompressed before doing the comparison)
# 
# (This is supposed to be a "prettier" replacement for runstdtest.)
#
#	Flags
#	~~~~~
#	-accept-output	replace output files with the ones actually generated by running
#			the program
#
($Pgm = $0) =~ s|.*/||;
$Verbose = 0;
$SaveStderr = 0;
$SaveStdout = 0;
$Status = 0;
@PgmArgs = ();
$PgmFail=0;
$PgmExitStatus = 0;
$PgmStdinFile  = '/dev/null';
if ( $ENV{'TMPDIR'} ) { # where to make tmp file names
    $TmpPrefix = $ENV{'TMPDIR'};
} else {
    $TmpPrefix ="$DEFAULT_TMPDIR";
    $ENV{'TMPDIR'} = "$DEFAULT_TMPDIR"; # set the env var as well
}
# If this is Cygwin, ignore eol and CR characters.
# Perhaps required for MSYS too, although the cygpath
# bit is hopefully unnecessary.
if ( `uname | grep CYGWIN` ) {
    $CONTEXT_DIFF=$CONTEXT_DIFF . " --strip-trailing-cr" ;
    $TmpPrefix = `cygpath -m $TmpPrefix | tr -d \\\\n`;
}
$ScriptFile = "$TmpPrefix/run_me$$";
$DefaultStdoutFile = "$TmpPrefix/no_stdout$$"; # can't use /dev/null (e.g. Alphas)
$DefaultStderrFile = "$TmpPrefix/no_stderr$$";
@PgmStdoutFile = ();
@PgmStderrFile = ();
$PreScript = '';
$PostScript = '';
$TimeCmd = '';
$StatsFile = "$TmpPrefix/stats$$";
$CachegrindStats = "cachegrind.out.summary";
$SysSpecificTiming = '';
$Cachegrind = 'no';

die "$Pgm: program to run not given as first argument\n" if $#ARGV < 0;
$ToRun = $ARGV[0]; shift(@ARGV);
# avoid picking up same-named thing from somewhere else on $PATH...
$ToRun = "./$ToRun" if -e "./$ToRun";

arg: while ($_ = $ARGV[0]) {
    shift(@ARGV);
    
    /^--$/	&& do { # let anything past after --
			push(@PgmArgs, @ARGV);
			last arg; };

    /^-v$/	       && do { $Verbose = 1; next arg; };
    /^-accept-output-stderr$/ && do { $SaveStderr = 1; next arg; };
    /^-accept-output-stdout$/ && do { $SaveStdout = 1; next arg; };
    /^-accept-output$/        && do { $SaveStdout = 1; $SaveStderr = 1; next arg; };

    /^-O(.*)/	&& do { push(@PgmArgs, &grab_arg_arg('-O',$1)); next arg; };
    /^-i(.*)/	&& do { $PgmStdinFile = &grab_arg_arg('-i',$1);
			$Status++,
			print STDERR "$Pgm: bogus -i input file: $PgmStdinFile\n"
			    if $PgmStdinFile !~ /^\/dev\/.*$/ && ! -f $PgmStdinFile;
			next arg; };
    /^-fail/    && do { $PgmFail=1; next arg; };
    /^-x(.*)/	&& do { $PgmExitStatus = &grab_arg_arg('-x',$1);
			$Status++ ,
			print STDERR "$Pgm: bogus -x expected exit status: $PgmExitStatus\n"
			    if $PgmExitStatus !~ /^\d+$/;
			next arg; };
    /^-o1(.*)/	&& do { $out_file = &grab_arg_arg('-o1',$1);
			push(@PgmStdoutFile, $out_file);
			next arg; };
    /^-o2(.*)/	&& do { $out_file = &grab_arg_arg('-o2',$1);
			push(@PgmStderrFile, $out_file);
			next arg; };
    /^-prescript(.*)/  && do { $PreScript = &grab_arg_arg('-prescript',$1);
			        next arg; };
    /^-postscript(.*)/ && do { $PostScript = &grab_arg_arg('-postscript',$1);
			        next arg; };
    /^-script/ && do { print STDERR "$Pgm: -script argument is obsolete;\nUse -prescript and -postscript instead.\n";
		    $Status++;
		    next arg; };
    /^-(ghc|hbc)-timing$/ && do { $SysSpecificTiming = $1;
				  next arg; };
    /^-cachegrind$/ && do { $SysSpecificTiming = 'ghc-instrs';
			    $Cachegrind = 'yes'; 
			    next arg };
    /^-t(.*)/	&& do { $TimeCmd = &grab_arg_arg('-t', $1); next arg; };

    # anything else is taken to be a pgm arg
    push(@PgmArgs, $_);
}

foreach $out_file ( @PgmStdoutFile ) {
    if ( ! -f $out_file && !$SaveStdout ) {
    	    print STDERR "$Pgm: warning: expected-stdout file missing: $out_file\n";
	    pop(@PgmStdoutFile);
    }
}

foreach $out_file ( @PgmStderrFile ) {
    if ( ! -f $out_file && !$SaveStderr ) {
   	    print STDERR "$Pgm: warning: expected-stderr file missing: $out_file\n";
	    pop(@PgmStderrFile);
    }
}

exit 1 if $Status;

# add on defaults if none specified
@PgmStdoutFile = ( $DefaultStdoutFile ) if $#PgmStdoutFile < 0;
@PgmStderrFile = ( $DefaultStderrFile ) if $#PgmStderrFile < 0;

# tidy up the pgm args:
# (1) look for the "first input file"
#     and grep it for "interesting" comments (-- !!! )
# (2) quote any args w/ whitespace in them.
$grep_done = 0;
foreach $a ( @PgmArgs ) {
    if (! $grep_done && $a !~ /^-/ && -f $a) {
	print `egrep "^--[ ]?!!!" $a`;
	$grep_done = 1;
    }
    if ($a =~ /\s/ || $a =~ /'/) {
	$a =~ s/'/\\'/g;    # backslash the quotes;
	$a = "\"$a\"";	    # quote the arg
    }
}

# deal with system-specific timing options
$TimingMagic = '';
if ( $SysSpecificTiming =~ /^ghc/ ) {
    $TimingMagic = "+RTS -S$StatsFile -RTS"
} elsif ( $SysSpecificTiming eq 'hbc' ) {
    $TimingMagic = "-S$StatsFile";
}

if ($PreScript ne '') {
    local($to_do);
    $PreScriptLines = `cat $PreScript`;
    $PreScriptLines =~ s/\r//g;
} else {
    $PreScriptLines = '';
}

if ($PostScript ne '') {
    local($to_do);
    $PostScriptLines = `cat $PostScript`;
    $PostScriptLines =~ s/\r//g;
    $* = 1;
    $PostScriptLines =~ s#\$o1#$TmpPrefix/runtest$$.1#g;
    $PostScriptLines =~ s#\$o2#$TmpPrefix/runtest$$.2#g;
} else {
    $PostScriptLines = '';
}

# OK, so we're gonna do the normal thing...

if ($Cachegrind eq 'yes') {
  $CachegrindPrefix = "valgrind --tool=cachegrind --log-fd=9 9>$CachegrindStats";
} else {
  $CachegrindPrefix = '';
}

$Script = <<EOSCRIPT;
#! /bin/sh
myexit=0
diffsShown=0
rm -f $DefaultStdoutFile $DefaultStderrFile
cat /dev/null > $DefaultStdoutFile
cat /dev/null > $DefaultStderrFile
$PreScriptLines
$SpixifyLine1
echo $TimeCmd /bin/sh -c \'$CachegrindPrefix $ToRun $TimingMagic @PgmArgs < $PgmStdinFile 1> $TmpPrefix/runtest$$.1 2> $TmpPrefix/runtest$$.2 3> $TmpPrefix/runtest$$.3\'
$TimeCmd /bin/sh -c \'$CachegrindPrefix $ToRun $TimingMagic @PgmArgs < $PgmStdinFile 1> $TmpPrefix/runtest$$.1 2> $TmpPrefix/runtest$$.2 3> $TmpPrefix/runtest$$.3\'
progexit=\$?
if [ \$progexit -eq 0 ] && [ $PgmFail -ne 0 ]; then
    echo $ToRun @PgmArgs \\< $PgmStdinFile
    echo "****" expected a failure, but was successful
    myexit=1
fi
if [ \$progexit -ne $PgmExitStatus ] && [ $PgmFail -eq 0 ]; then
    echo $ToRun @PgmArgs \\< $PgmStdinFile
    echo "****" expected exit status $PgmExitStatus not seen \\; got \$progexit
    myexit=1
else
    $PostScriptLines
    hit='NO'
    for out_file in @PgmStdoutFile ; do
	if cmp -s \$out_file $TmpPrefix/runtest$$.1 ; then
	    hit='YES'
	fi
    done
    if [ \$hit = 'NO' ] ; then
	echo $ToRun @PgmArgs \\< $PgmStdinFile
	echo expected stdout not matched by reality
	orig_file="$PgmStdoutFile[0]";
	[ ! -f \$orig_file ] && orig_file="/dev/null"
	${CONTEXT_DIFF} \$orig_file $TmpPrefix/runtest$$.1
	myexit=\$?
	diffsShown=1
    fi
    if [ $SaveStdout = 1 ] && 
       [ $PgmStdoutFile[0] != $DefaultStdoutFile ] && [ -s $TmpPrefix/runtest$$.1 ]; then
	echo Saving away stdout output in $PgmStdoutFile[0] ...
	if [ -f $PgmStdoutFile[0] ]; then
	     rm -f $PgmStdoutFile[0].bak
	     cp $PgmStdoutFile[0] $PgmStdoutFile[0].bak
	fi;
	cp $TmpPrefix/runtest$$.1 $PgmStdoutFile[0]
    fi
fi

hit='NO'
for out_file in @PgmStderrFile ; do
    if cmp -s \$out_file $TmpPrefix/runtest$$.2 ; then
	hit='YES'
    fi
done
if [ \$hit = 'NO' ] ; then
    echo $ToRun @PgmArgs \\< $PgmStdinFile
    echo expected stderr not matched by reality
    orig_file="$PgmStderrFile[0]"
    [ ! -f \$orig_file ] && orig_file="/dev/null"
    ${CONTEXT_DIFF} \$orig_file $TmpPrefix/runtest$$.2
    myexit=\$?
    diffsShown=1
fi
if [ $SaveStderr = 1 ] &&
   [ $PgmStderrFile[0] != $DefaultStderrFile ] && [ -s $TmpPrefix/runtest$$.2 ]; then
	echo Saving away stderr output in $PgmStderrFile[0] ...
	if [ -f $PgmStderrFile[0] ]; then
	   rm -f $PgmStderrFile[0].bak
	   cp $PgmStderrFile[0] $PgmStderrFile[0].bak
	fi;
	cp $TmpPrefix/runtest$$.2 $PgmStderrFile[0]
fi

${RM} core $ToRunOrig.spix $DefaultStdoutFile $DefaultStderrFile $TmpPrefix/runtest$$.1 $TmpPrefix/runtest$$.2 $TmpPrefix/runtest$$.3
exit \$myexit
EOSCRIPT

# bung script into a file
open(SCR, "> $ScriptFile") || die "Failed opening script file $ScriptFile!\n";
print SCR $Script;
close(SCR) || die "Failed closing script file!\n";
chmod 0755, $ScriptFile;

print STDERR $Script if $Verbose;

&run_something($ScriptFile);

if ( $SysSpecificTiming eq '' ) {
    unlink $StatsFile;
    unlink $ScriptFile;
    exit 0;
}

&process_stats_file();
&process_cachegrind_files() if $Cachegrind eq 'yes';

# print out what we found
print STDERR "<<$SysSpecificTiming: ";
if ( $Cachegrind ne 'yes') {
	print STDERR "$BytesAlloc bytes, $GCs GCs, $AvgResidency/$MaxResidency avg/max bytes residency ($ResidencySamples samples), $GCWork bytes GC work, ${TotMem}M in use, $InitTime INIT ($InitElapsed elapsed), $MutTime MUT ($MutElapsed elapsed), $GcTime GC ($GcElapsed elapsed)";
} else {
	print STDERR "$BytesAlloc bytes, $GCs GCs, $AvgResidency/$MaxResidency avg/max bytes residency ($ResidencySamples samples), $GCWork bytes GC work, ${TotMem}M in use, $InitTime INIT ($InitElapsed elapsed), $MutTime MUT ($MutElapsed elapsed), $GcTime GC ($GcElapsed elapsed), $TotInstrs instructions, $TotReads memory reads, $TotWrites memory writes, $TotMisses L2 cache misses";
};
print STDERR " :$SysSpecificTiming>>\n";

# OK, party over
unlink $StatsFile;
unlink $ScriptFile;
exit 0;

sub grab_arg_arg {
    local($option, $rest_of_arg) = @_;
    
    if ($rest_of_arg ne "") {
	return($rest_of_arg);
    } elsif ($#ARGV >= 0) {
	local($temp) = $ARGV[0]; shift(@ARGV); 
	return($temp);
    } else {
	print STDERR "$Pgm: no argument following $option option\n";
	$Status++;
    }
}

sub run_something {
    local($str_to_do) = @_;

#   print STDERR "$str_to_do\n" if $Verbose;

    local($return_val) = 0;
    system($str_to_do);
    $return_val = $?;

    if ($return_val != 0) {
#ToDo: this return-value mangling is wrong
#	local($die_msg) = "$Pgm: execution of the $tidy_name had trouble";
#	$die_msg .= " (program not found)" if $return_val == 255;
#	$die_msg .= " ($!)" if $Verbose && $! != 0;
#	$die_msg .= "\n";
	unlink $ScriptFile;
	unlink $StatsFile;

	exit (($return_val == 0) ? 0 : 1);
    }
}

sub process_stats_file {

    # OK, process system-specific stats file
    if ( $SysSpecificTiming =~ /^ghc/ ) {

	#NB: nearly the same as in GHC driver's -ghc-timing stuff

	open(STATS, $StatsFile) || die "Failed when opening $StatsFile\n";

	local($max_live)    = 0; 
	local($tot_live)    = 0; # for calculating residency stuff
	local($tot_samples) = 0;

	$GCWork = 0;
	while (<STATS>) {
	    if (! /Gen:\s+0/ && /^\s*\d+\s+\d+\s+(\d+)\s+\d+\.\d+/ ) {
		$max_live = $1 if $max_live < $1;
		$tot_live += $1;
		$tot_samples += 1;
	    }

	    $BytesAlloc = $1 if /^\s*([0-9,]+) bytes allocated in the heap/;
	    $GCWork += $1    if /^\s*([0-9,]+) bytes copied during GC/;

#	    if ( /^\s*([0-9,]+) bytes maximum residency .* (\d+) sample/ ) {
#		$MaxResidency = $1; $ResidencySamples = $2;
#	    }

	    $GCs = $1 if /^\s*([0-9,]+) collections? in generation 0/;

	    if ( /^\s+([0-9]+)\s+Mb total memory/ ) {
		$TotMem = $1;
	    }

	    if ( /^\s*INIT\s+time\s*(-*\d+\.\d\d)s\s*\(\s*(-*\d+\.\d\d)s elapsed\)/ ) {
		$InitTime = $1; $InitElapsed = $2;
	    } elsif ( /^\s*MUT\s+time\s*(-*\d+\.\d\d)s\s*\(\s*(-*\d+\.\d\d)s elapsed\)/ ) {
		$MutTime = $1; $MutElapsed = $2;
	    } elsif ( /^\s*GC\s+time\s*(-*\d+\.\d\d)s\s*\(\s*(-*\d+\.\d\d)s elapsed\)/ ) {
		$GcTime = $1; $GcElapsed = $2;
	    }
	}
	close(STATS) || die "Failed when closing $StatsFile\n";
	if ( $tot_samples > 0 ) {
	    $ResidencySamples = $tot_samples;
	    $MaxResidency = $max_live;
	    $AvgResidency = int ($tot_live / $tot_samples) ;
	}

    } elsif ( $SysSpecificTiming eq 'hbc' ) {

	open(STATS, $StatsFile) || die "Failed when opening $StatsFile\n";
	while (<STATS>) {
	    $BytesAlloc = $1 if /^\s*([0-9]+) bytes allocated from the heap/;

	    $GCs = $1 if /^\s*([0-9]+) GCs?,$/;

	    if ( /^\s*(\d+\.\d\d) \((\d+\.\d)\) seconds total time,$/ ) {
		$MutTime = $1; $MutElapsed = $2; # will fix up later

		$InitTime = 0; $InitElapsed = 0; # hbc doesn't report these

	    } elsif ( /^\s*(\d+\.\d\d) \((\d+\.\d)\) seconds GC time/ ) {
		$GcTime = $1; $GcElapsed = $2;

		# fix up mutator time now
		$MutTime    = sprintf("%.2f", ($MutTime    - $GcTime));
		$MutElapsed = sprintf("%.1f", ($MutElapsed - $GcElapsed));
	    }
	}
	close(STATS) || die "Failed when closing $StatsFile\n";
    }

    # warn about what we didn't find
    print STDERR "Warning: BytesAlloc not found in stats file\n" unless defined($BytesAlloc);
    print STDERR "Warning: GCs not found in stats file\n" unless defined($GCs);
    print STDERR "Warning: InitTime not found in stats file\n" unless defined($InitTime);
    print STDERR "Warning: InitElapsed not found in stats file\n" unless defined($InitElapsed);
    print STDERR "Warning: MutTime not found in stats file\n" unless defined($MutTime);
    print STDERR "Warning: MutElapsed not found in stats file\n" unless defined($MutElapsed);
    print STDERR "Warning: GcTime inot found in stats file\n" unless defined($GcTime);
    print STDERR "Warning: GcElapsed not found in stats file\n" unless defined($GcElapsed);
    print STDERR "Warning: total memory not found in stats file\n" unless defined($TotMem);
    print STDERR "Warning: GC work not found in stats file\n" unless defined($GCWork);

    # things we didn't necessarily expect to find
    $MaxResidency     = 0 unless defined($MaxResidency);
    $AvgResidency     = 0 unless defined($AvgResidency);
    $ResidencySamples = 0 unless defined($ResidencySamples);

    # a bit of tidying
    $BytesAlloc =~ s/,//g;
    $GCWork =~ s/,//g;
    $MaxResidency =~ s/,//g;
    $GCs =~ s/,//g;
    $InitTime =~ s/,//g;
    $InitElapsed =~ s/,//g;
    $MutTime =~ s/,//g;
    $MutElapsed =~ s/,//g;
    $GcTime =~ s/,//g;
    $GcElapsed =~ s/,//g;
}

sub process_cachegrind_files {

    open(STATS, "< $CachegrindStats") || die("Can't open $CachegrindStats\n");

    while (<STATS>) {
	/^==\d+==\s+I\s+refs:\s+([0-9,]*)/ && do {
	   $TotInstrs = $1;
	   $TotInstrs =~ s/,//g;
	};

	/^==\d+==\s+D\s+refs:\s+[0-9,]+\s+\(([0-9,]+)\s+rd\s+\+\s+([0-9,]+)\s+wr/ && do {
	   $TotReads  = $1;
	   $TotWrites = $2;
	   $TotReads  =~ s/,//g;
	   $TotWrites =~ s/,//g;
	};

	/^==\d+==\s+L2d\s+misses:\s+([0-9,]+)/ && do {
	   $TotMisses = $1;
	   $TotMisses =~ s/,//g;
	};
    }
    close(STATS);
}