From 1e7603ac5a3cca436d75ab0ee0778ec18919cbd8 Mon Sep 17 00:00:00 2001 From: Nick Bowler Date: Thu, 2 Jul 2020 23:16:51 -0400 Subject: [PATCH] Port random crossparse test to Autotest. I don't really know why the existing crossparse testcase is so complicated. Sure running the test generation and execution in parallel may be an interesting approach, but it seems to be total overkill for this use case. By enhancing the crossparse application to take a list of test cases from a file, we can just generate the stimuli in one step and execute the tests in another which is simple and works fine. As this is the final test to port to Autotest, we can now retire the use of the Automake test harness. --- Makefile.am | 9 ---- test/crossparse.c | 56 ++++++++++++++++++----- tests/crossparse-c-random.sh | 89 ------------------------------------ tests/stress.at | 10 ++++ testsuite.at | 7 +++ 5 files changed, 61 insertions(+), 110 deletions(-) delete mode 100755 tests/crossparse-c-random.sh diff --git a/Makefile.am b/Makefile.am index 8ccdc95..da056d7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4,7 +4,6 @@ # This is free software: you are free to do what the fuck you want to. # There is NO WARRANTY, to the extent permitted by law. -AUTOMAKE_OPTIONS = parallel-tests color-tests ACLOCAL_AMFLAGS = -I m4 -I common/m4 # For Gnulib @@ -72,14 +71,6 @@ $(test_crossparse_OBJECTS): $(gnulib_headers) test_randomdecl_LDADD = libcdecl.la libtest.la libgnu.la $(test_randomdecl_OBJECTS): $(gnulib_headers) -TESTS_ENVIRONMENT = SHELL='$(SHELL)' LIBTOOL='$(LIBTOOL)' EXEEXT='$(EXEEXT)' -TEST_EXTENSIONS = .sh -SH_LOG_COMPILER = $(SHELL) - -TESTS = tests/crossparse-c-random.sh - -dist_check_SCRIPTS = $(TESTS) - src/parse.lo: src/scan.h src/scan.lo: src/parse.h src/parse-decl.lo: src/scan.h src/parse.h diff --git a/test/crossparse.c b/test/crossparse.c index 05c3095..210714a 100644 --- a/test/crossparse.c +++ b/test/crossparse.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -27,10 +28,11 @@ #define PROGNAME "crossparse" static const char *progname = PROGNAME; -static const char sopts[] = "ECVH"; +static const char sopts[] = "f:ECVH"; static const struct option lopts[] = { { "cdecl", 0, NULL, 'C' }, { "english", 0, NULL, 'E' }, + { "file", 2, NULL, 'f' }, { "version", 0, NULL, 'V' }, { "help", 0, NULL, 'H' }, { 0 } @@ -131,13 +133,22 @@ out: free(buf1); free(buf2); free(buf3); + + if (!ret) { + fprintf(stderr, "%s: failed cross-parse check of: %s\n", + progname, str); + } + return ret; } int main(int argc, char **argv) { int opt, mode = MODE_CDECL; - int ret = EXIT_FAILURE; + int ret = EXIT_SUCCESS; + + const char *filename = NULL; + FILE *infile = NULL; if (argc > 0) progname = argv[0]; @@ -150,6 +161,10 @@ int main(int argc, char **argv) case 'E': mode = MODE_ENGLISH; break; + case 'f': + infile = stdin; + filename = optarg; + break; case 'V': test_print_version(PROGNAME); return EXIT_SUCCESS; @@ -162,18 +177,35 @@ int main(int argc, char **argv) } } - if (!argv[optind]) { - print_usage(stderr); - return EXIT_FAILURE; - } + if (infile) { + char *line = NULL; + size_t n; + + if (filename) { + infile = fopen(filename, "r"); + if (!infile) { + fprintf(stderr, "%s: %s: %s\n", progname, + filename, strerror(errno)); + return EXIT_FAILURE; + } + } - for (int i = optind; i < argc; i++) { - if (!test_crossparse(argv[i], mode)) { - fprintf(stderr, "%s: failed cross-parse check of: %s\n", - progname, argv[i]); - return EXIT_FAILURE; + while (getline(&line, &n, infile) >= 0) { + char *c = strchr(line, '\n'); + if (c) + *c = '\0'; + if (!test_crossparse(line, mode)) + ret = EXIT_FAILURE; } + } else if (argv[optind]) { + for (int i = optind; i < argc; i++) { + if (!test_crossparse(argv[i], mode)) + ret = EXIT_FAILURE; + } + } else { + print_usage(stderr); + return EXIT_FAILURE; } - return EXIT_SUCCESS; + return ret; } diff --git a/tests/crossparse-c-random.sh b/tests/crossparse-c-random.sh deleted file mode 100755 index a800f3f..0000000 --- a/tests/crossparse-c-random.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/sh -# -# Copyright © 2012 Nick Bowler -# -# Randomized tests that libcdecl can parse its own output. -# -# License WTFPL2: Do What The Fuck You Want To Public License, version 2. -# This is free software: you are free to do what the fuck you want to. -# There is NO WARRANTY, to the extent permitted by law. - -: "${RANDOMSEED=$RANDOM}" "${TESTITER=500}" - -randomdecl=test/randomdecl$EXEEXT -crossparse=test/crossparse$EXEEXT -test -x $randomdecl || exit 77 - -# Slow case: run tests one at a time to determine exactly which one failed. -first_failed() { - ff_count=0 - - while test $# -gt 0 - do - $crossparse "$1" 2>/dev/null || break - ff_count=`expr $ff_count + 1` - shift - done - - echo "$ff_count" -} - -proc() { - result=pass - count=0 - - set x - while read decl - do - set "$@" "$decl" - - # Accumulate tests in $@ and run them in batches to avoid - # significant startup costs. - if test $# -gt 25; then - shift - $crossparse "$@" || { result=fail - tmp_count=`first_failed "$@"` - count=`expr $count + $tmp_count` - break - } - count=`expr $count + $#` - set x - fi - done - - shift - if test $# -gt 0 && test x"$result" = x"pass"; then - tmp_count=$# - $crossparse "$@" || { result=fail - tmp_count=`first_failed "$@"` - } - count=`expr $count + $tmp_count` - fi - - echo "result=$result" - echo "count=$count" -} - -printf '%s: randomized test using RANDOMSEED=%d\n' "$0" "$RANDOMSEED" - -eval_cmd=`exec 3>&1 - { $randomdecl -s "$RANDOMSEED" -n "$TESTITER" 3>&- - echo gen_status=$? >&3 - } | proc >&3` -eval "$eval_cmd" - -if test x"$gen_status" != x"0"; then - printf '%s: %s failed with status=%d\n' \ - "$0" "$randomdecl" "$gen_status" 1>&2 - exit 1 -fi - -expected_count=`expr "$TESTITER" + 0` -if test x"$count" != x"$expected_count"; then - printf '%s: failed after %d successful tests (out of %d)\n' \ - "$0" "$count" "$expected_count" 1>&2 - exit 1 -fi - -test x"$result" = x"pass" && exit 0 -exit 1 diff --git a/tests/stress.at b/tests/stress.at index 9b45cea..77c0d20 100644 --- a/tests/stress.at +++ b/tests/stress.at @@ -79,3 +79,13 @@ m4_define([sanity_check], [AT_CHECK([$AS_TR_SH([$2]) false])]) m4_map([sanity_check], [sanity_tests]) AT_CLEANUP + +AT_SETUP([random cross-parse]) + +TEST_NEED_PROGRAM([randomdecl]) +TEST_NEED_PROGRAM([crossparse]) + +printf 'Using seed %d\n' "$random_seed" >&AS_MESSAGE_LOG_FD +AT_CHECK([randomdecl -n "$random_iter" -s "$random_seed"],, [stdout-nolog]) +AT_CHECK([crossparse -f stdout]) +AT_CLEANUP diff --git a/testsuite.at b/testsuite.at index 930ff2c..2fb25be 100644 --- a/testsuite.at +++ b/testsuite.at @@ -23,10 +23,17 @@ AT_ARG_OPTION_ARG([random-seed], [Integer seed for randomized testing (default: auto)])], [random_seed=$at_optarg], [random_seed=$RANDOM$$]) +AT_ARG_OPTION_ARG([random-iterations], + [AS_HELP_STRING([--random-iterations=N], + [Number of randomized testcases to perform (default: 500)])], + [random_iter=$at_optarg], [random_iter=]) + m4_divert_push([PREPARE_TESTS])dnl : ${LIBTOOL="$SHELL $builddir/libtool"} random_seed=`printf '%s' "$random_seed" | tr -cd '@<:@:digit:@:>@'` random_seed=${random_seed:-0} +random_iter=`printf '%s' "$random_iter" | tr -cd '@<:@:digit:@:>@'` +random_iter=`expr "$random_iter" '|' 500` m4_divert_pop([PREPARE_TESTS]) m4_define([TEST_NEED_PROGRAM], [AT_KEYWORDS([$1])dnl -- 2.43.2