Re: RFA: Don't include value of expression in pc-fp.exp test name

[Andrew Cagney <ac131313@redhat.com>]

[-- Attachment #1: Type: text/plain, Size: 1615 bytes --]

> ac> As Michael well knows, supplemental information, such as which specific 
> ac> branch of a test passed or failed can be included in paren in the test 
> ac> message.  Any analysis tools comparing test results needs to accomodate 
> ac> this convention.
> 
> I don't know any such convention.

Michael, you and I had an e-mail exchange about this very issue.  The 
end result, last time, was no change.

> I'm looking at a typical gdb.sum.
> There are 1523 instances of '(...)' in 112 different test scripts.
> Most of these are part of the test name.

To split hairs, I can see two cases:

- two runs within identical environments
I can see 'diff -u' reasonably working here.

- two runs within different environments
After paren stripping the results should be identical (or close two it).

For instance sizeof.exp contains various tests to check that sizes are 
sane.  The actual sizes found are included in the output.  That's fine 
since if the numbers were to change between runs the test results are 
pretty sunk.

Thanks to the advent of PIE (position independant code) it's now 
possible for the PC/FP to change between "100% identical" runs.  So for 
this specific case, what ever.

David, I've attached a script (repost).  It's what Michael tables 
started out using but have likely since been completly rewritten.

Andrew


> If you want to promulgate such a convention, put it in the docs,
> and I'll start submitting patches to change all these test scripts,
> or committing them as obvious fixes.
> 
> A better path would be to pick a character sequence which is unused,
> such as '// ...'.
> 


[-- Attachment #2: do-analize-tests --]
[-- Type: text/plain, Size: 7904 bytes --]

#!/bin/sh

if awk --version < /dev/null 2>&1 | grep -i gnu > /dev/null
then
    AWK=awk
else
    AWK=gawk
fi

$AWK '
function trace (msg) {
    print msg >> "/dev/stderr"
}


# returns a SUPSEP separated list created by sorting TABLE.
function qsort (table, middle, tmp, left, nr_left, right, nr_right, result) {
    middle = ""
    for (middle in table) { break; }
    nr_left = 0;
    nr_right = 0;
    for (tmp in table) {
	if (tmp < middle) {
	    nr_left++
	    left[tmp] = tmp
	} else if (tmp > middle) {
	    nr_right++
	    right[tmp] = tmp
	}
    }
    #print "qsort " nr_left " " middle " " nr_right > "/dev/stderr"
    result = ""
    if (nr_left > 0) {
	result = qsort(left) SUBSEP
    }
    result = result middle
    if (nr_right > 0) {
	result = result SUBSEP qsort(right)
    }
    return result
}


function print_headings (html_p, file, title, target, i) {
    if (html_p) {
	printf "<tr><td></td><th>"					>> file
    }
    printf title							>> file
    if (html_p) {
	printf "</th>"							>> file
    }
    for (i = 1; i <= nr_targets; i++) {
	target = sorted_targets[i]
	if (html_p) {
	    printf "<th>" i "<br>" target "</th>"			>> file
	} else {
	    printf "\t" i						>> file
	}
    }
    if (html_p) {
	printf "</tr>"							>> file
    }
    printf "\n"								>> file
}

function print_test (file, test, nr, i) {
    printf "<tr>"							>> file
    printf "<td>%d</td>", nr						>> file
    printf "<th align=left>%s</th>", test				>> file
    for (i = 1; i <= nr_targets; i++) {
	target = sorted_targets[i];
	result = results[target, test]
	if (result == "FAIL") {
	    printf "<th>%s</th>", result				>> file
	} else {
	    printf "<td align=center>%s</td>", result			>> file
	}
    }
    printf "</tr>\n"							>> file
}

function print_totals (html_p, file, status, target, i, j) {
    for (i = 1; i <= nr_statuses; i++) {
	status = sorted_statuses[i];
	if (html_p) {
	    printf "<tr><td></td><th align=left>"			>> file
	}
	printf status							>> file
	if (html_p) {
	    printf "</th>"						>> file
	}
	for (j = 1; j <= nr_targets; j++) {
	    target = sorted_targets[j];
	    if (html_p) {
		printf "<th>"						>> file
	    } else {
		printf "\t"
	    }
	    printf totals[target, status]				>> file
	    if (html_p) {
		printf "</th>"						>> file
	    }
	}
	if (html_p) {
	    printf "</tr>"						>> file
	}
	printf "\n"							>> file
    }
}

function print_contents (html_p, file, i, j, target, status) {
    if (html_p) print "<table>"						>> file
    if (html_p) printf "<tr><th>"     	                                >> file
    printf "Index"			                                >> file
    if (html_p) printf "</th>"	   	                                >> file
    for (j = 1; j <= nr_statuses; j++) {
	status = sorted_statuses[j];
	if (html_p) printf "<th align=right>"	       	          	>> file
	printf "\t%s", status			                        >> file
	if (html_p) printf "</th>"		                        >> file
    }
    if (html_p) printf "<th>Variant</th><th>File</th></tr>"		>> file
    printf "\n"								>> file
    if (html_p) printf "<tr><th></th>"					>> file
    for (i = 1; i <= nr_targets; i++) {
	target = sorted_targets[i]
	if (html_p) printf "<tr>"					>> file
	if (html_p) printf "<td align=right>"				>> file
	printf "%d", i							>> file
	if (html_p) printf "</td>"					>> file
	for (j = 1; j <= nr_statuses; j++) {
	    status = sorted_statuses[j];
	    if (html_p) printf "<td align=right>"			>> file
	    printf "\t%d", totals[target, status]			>> file
	    if (html_p) printf "</td>"					>> file
	}
	if (html_p) printf "<td>"					>> file
	printf "\t%s", target						>> file
	if (html_p) printf "</td>"					>> file
	if (html_p) printf "<td>"					>> file
	printf " %s", target_file[target]				>> file
	if (html_p) printf "</td>"					>> file
	if (html_p) printf "</tr>"					>> file
	printf "\n"							>> file
    }
    if (html_p) print "</table>"					>> file
}


function print_begin (file, title, full_title, i, j, target, status) {
    full_title = "Failure Table: " title;
    trace("Generating " file " (" full_title ")")
    printf "" > file
    print "<html>"							>> file
    print "<head>"							>> file
    print "<title>" full_title "</title>"				>> file
    print "</head>"							>> file

    print "<body>"							>> file
    print "<h1><center>" full_title "</center></h1>"			>> file
    print "<center>"							>> file
    print_contents(1, file)
    print "</center>"							>> file
}

function print_end (file) {
    print "</body>"							>> file
    print "</html>"							>> file
}

function print_opening (file, title) {
    print_begin(file, title);
    print "<table>"							>> file
    print_headings(1, file, title);
    print_totals(1, file);
}

function print_closing (file, title, nr) {
    trace(title ": " nr);
    print_totals(1, file);
    print_headings(1, file, title);
    print "</table>"							>> file
    print_end(file);
}

BEGIN {
    nr_runs = 0
}
/Running target/ {
    nr_runs++
    sub (/Running target/, "");
    gsub (/ /, "");
    gsub (/\//, " ");
    nr = 1;
    do {
	target = $0 " (" nr++ ")";
    } while (target in targets)
    targets[target] = target
    target_file[target] = FILENAME
    trace("Parsing " nr_runs ": " target " " (target_file[target]));
    next
}

# Collect up sizeof???

# Collect the results (does this work?)
/^((|X|K)(PASS|FAIL)|UNRESOLVED|ERROR|WARNING):/ {
    # accumulate statuses
    status = $0
    sub (/:.*$/, "", status)
    statuses[status] = status
    # accumulate tests
    test = $0
    sub (/^[A-Z]*: /, "", test)
    gsub (/ *\([^)]*\)/, "", test)
    gsub (/ \/[^ ]*\/gdb\//, " ..../gdb/", test)
    tests[test] = test

    # accumulate result counts
    totals[target, status] += 1;
    results[target, test] = status;
    totals[test, status] += 1;
}


END {

    # sort the test names
    trace("Sorting results")
    nr_tests = split (qsort(tests), sorted_tests, SUBSEP);
    trace("Number of tests " nr_tests)
#    for (i = 0; i < nr_tests; i++) {
#	print i " " sorted_tests[i] > "/dev/stderr"
#    }
    nr_targets = split (qsort(targets), sorted_targets, SUBSEP);
    nr_statuses = split (qsort(statuses), sorted_statuses, SUBSEP);

    print_contents(0, "/dev/stderr")
    print_headings(0, "/dev/stderr", "Summary")
    print_totals(0, "/dev/stderr")

    ###

    file = "index.html"
    print_begin(file, "Contents")
    print "<center>"							>> file
    print "<a href=pass.html>Passes</a>"				>> file
    print "<a href=fail.html>Failures</a>"				>> file
    print "<a href=diff.html>Differences</a>"				>> file
    print "</center>"							>> file
    print_end(file)

    ####

    # heading
    file = "diff.html"
    print_opening(file, "Differences");

    # body - differences
    nr = 0
    for (i = 1; i <= nr_tests; i++) {
	test = sorted_tests[i];
	different = 0;
	fail = 0;
	result = "<uninitialized>";
	for (target in targets) {
	    if (result == "<uninitialized>") {
		result = results[target, test];
	    } else if (result != results[target, test]) {
		different = 1
	    }
	    if (results[target, test] == "FAIL") {
		fail = 1;
	    }
#	    print "TEST=" test, "different=" different, "fail=" fail, "result=" result > "/dev/stderr"
	}
	if (different) {
	    print_test(file, test, ++nr);
	}
    }

    print_closing(file, "Differences", nr);

    ####

    file = "fail.html"
    print_opening(file, "Failures");

    # body - failures
    nr = 0
    for (i = 1; i <= nr_tests; i++) {
	test = sorted_tests[i]
	if (totals[test, "FAIL"] > 0) {
	    print_test(file, test, ++nr);
	}
    }

    print_closing(file, "Failures", nr);

    ####

    file = "pass.html"
    print_opening(file, "Passes");

    # body - passes
    nr = 0
    for (i = 1; i <= nr_tests; i++) {
	test = sorted_tests[i]
	if (totals[test, "PASS"] > 0) {
	    print_test(file, test, ++nr);
	}
    }

    print_closing(file, "Passes", nr);

}
' "$@"