It's nice to see that the new TAP support in Automake, albeit still incomplete, can work correctly with the testsuite of an important, real-world package like Git. The attached patch, to be applied to the maint branch of git (at the moment of writing, that is commit `v1.7.6-43-g0906f6e'), should demonstrate this.
How to reproduce: $ cd /tmp $ [save the attached patch in /tmp] $ git clone git://git.sv.gnu.org/automake.git $ cd automake $ git checkout origin/test-protocols # Should be commit `v1.11-1019-ga3e7e1b' $ ./bootstrap && ./configure --prefix=`pwd`/_inst && make install $ export PATH=`pwd`/_inst/bin:$PATH $ cd .. $ git clone git://git.kernel.org/pub/scm/git/git.git $ cd git $ git checkout origin/maint # Should be commit `v1.7.6-43-g0906f6e' $ git am -3 /tmp/0001-allow-automake-TAP-driver-to-run-the-git-testsuite.patch $ make autotoolize $ ./configure && make $ cd t $ make check -j3 use_automake_test_harness=yes # Have fun! make check-TESTS make[1]: Entering directory `/tmp/git/t' make[2]: Entering directory `/tmp/git/t' PASS: t0000-basic.sh 1 - .git/objects should be empty after git init in an empty repo. PASS: t0002-gitfile.sh 1 - initial setup PASS: t0000-basic.sh 2 - .git/objects should have 3 subdirectories. PASS: t0000-basic.sh 3 - success is reported like this XFAIL: t0000-basic.sh 4 - pretend we have a known breakage # TODO known breakage PASS: t0002-gitfile.sh 2 - bad setup: invalid .git file format PASS: t0001-init.sh 1 - plain PASS: t0002-gitfile.sh 3 - bad setup: invalid .git file path PASS: t0002-gitfile.sh 4 - final setup + check rev-parse --git-dir PASS: t0002-gitfile.sh 5 - check hash-object PASS: t0002-gitfile.sh 6 - check cat-file PASS: t0000-basic.sh 5 - pretend we have fixed a known breakage (run in sub test-lib) PASS: t0000-basic.sh 6 - test runs if prerequisite is satisfied SKIP: t0000-basic.sh 7 # SKIP unmet prerequisite causes test to be skipped (missing DONTHAVEIT) PASS: t0000-basic.sh 8 - test runs if prerequisites are satisfied SKIP: t0000-basic.sh 9 # SKIP unmet prerequisites causes test to be skipped (missing DONTHAVEIT of HAVEIT,DONTHAVEIT) SKIP: t0000-basic.sh 10 # SKIP unmet prerequisites causes test to be skipped (missing DONTHAVEIT of DONTHAVEIT,HAVEIT) PASS: t0000-basic.sh 11 - tests clean up after themselves PASS: t0001-init.sh 2 - plain nested in bare PASS: t0002-gitfile.sh 7 - check update-index PASS: t0002-gitfile.sh 8 - check write-tree PASS: t0001-init.sh 3 - plain through aliased command, outside any git repo PASS: t0002-gitfile.sh 9 - check commit-tree XFAIL: t0001-init.sh 4 - plain nested through aliased command # TODO known breakage PASS: t0002-gitfile.sh 10 - check rev-list ... And then, wanna see the test harness report some failures and errors, and maybe some TAP diagnostic? Here we go ... $ sed -i '51s/test_expect_success/test_expect_failure/' t0000-basic.sh $ sed -i '54s/test_expect_failure/test_expect_success/' t0000-basic.sh $ sed -i 's|^test_done$|(exit 23); exit 23; &|' t0001-init.sh $ sed -i '1s|^#!/bin/sh$|&\necho Bail out!|' t9800-git-p4.sh $ make check \ > use_automake_test_harness=yes \ > SH_LOG_DRIVER_FLAGS=--comments \ > TESTS='t9800-git-p4.sh t0001-init.sh t0000-basic.sh' make check-TESTS make[1]: Entering directory `/tmp/git/t' make[2]: Entering directory `/tmp/git/t' ERROR: t9800-git-p4.sh - Bail out! PASS: t0001-init.sh 1 - plain ... PASS: t0000-basic.sh 48 - very long name in the index handled sanely # t0000-basic.sh: fixed 1 known breakage(s) # t0000-basic.sh: failed 2 among 48 test(s) ============================================================================ Testsuite summary for git 1.7.6.44.g08723 ============================================================================ # TOTAL: 86 # PASS: 75 # SKIP: 3 # XFAIL: 2 # FAIL: 2 # XPASS: 1 # ERROR: 3 ============================================================================ See t/test-suite.log Please report to g...@vger.kernel.org ============================================================================ Regards, Stefano
From 4490dc1b29d9095d877a3fe82ac26fa9e0fc0c24 Mon Sep 17 00:00:00 2001 Message-Id: <4490dc1b29d9095d877a3fe82ac26fa9e0fc0c24.1313332167.git.stefano.lattar...@gmail.com> From: Stefano Lattarini <stefano.lattar...@gmail.com> Date: Sun, 14 Aug 2011 11:06:05 +0200 Subject: [PATCH] allow automake TAP driver to run the git testsuite --- .gitignore | 3 + Makefile | 40 +++- aclocal.m4 => acinclude.m4 | 0 configure.ac | 13 +- t/.gitignore | 4 + t/Makefile | 6 + t/Makefile.auto.am | 591 ++++++++++++++++++++++++++++++++++++++++++++ t/tap-driver | 451 +++++++++++++++++++++++++++++++++ t/test-lib.sh | 6 +- 9 files changed, 1100 insertions(+), 14 deletions(-) rename aclocal.m4 => acinclude.m4 (100%) create mode 100644 t/Makefile.auto.am create mode 100755 t/tap-driver diff --git a/.gitignore b/.gitignore index acffdfa..117d331 100644 --- a/.gitignore +++ b/.gitignore @@ -209,6 +209,9 @@ /config.mak.autogen /config.mak.append /configure +/aclocal.m4 +/install-sh +/missing /tags /TAGS /cscope* diff --git a/Makefile b/Makefile index e40ac0c..0b69cc0 100644 --- a/Makefile +++ b/Makefile @@ -251,6 +251,11 @@ all:: # dependency rules. # # Define NATIVE_CRLF if your platform uses CRLF for line endings. +# + +ACLOCAL = aclocal +AUTOCONF = autoconf +AUTOMAKE = automake GIT-VERSION-FILE: FORCE @$(SHELL_PATH) ./GIT-VERSION-GEN @@ -1818,12 +1823,23 @@ $(patsubst %.py,%,$(SCRIPT_PYTHON)): % : unimplemented.sh mv $@+ $@ endif # NO_PYTHON -configure: configure.ac - $(QUIET_GEN)$(RM) $@ $<+ && \ - sed -e 's/@@GIT_VERSION@@/$(GIT_VERSION)/g' \ - $< > $<+ && \ - autoconf -o $@ $<+ && \ - $(RM) $<+ +configure_deps = acinclude.m4 aclocal.m4 configure.ac GIT-VERSION-FILE + +aclocal.m4: configure.ac GIT-VERSION-FILE + $(ACLOCAL) +configure: $(configure_deps) + $(AUTOCONF) +config.status: configure + ./config.status --recheck +t/Makefile.auto: config.status t/Makefile.auto.in + ./config.status t/Makefile.auto +t/Makefile.auto.in: t/Makefile.auto.am $(configure_deps) + $(AUTOMAKE) --add-missing --copy t/Makefile.auto + +autotoolize: aclocal.m4 configure t/Makefile.auto.in +# For compatibility with automake-generated remake rules. +am--refresh: autotoolize t/Makefile.auto +.PHONY: autotoolize am--refresh # These can record GIT_VERSION git.o git.spec \ @@ -2344,8 +2360,14 @@ dist-doc: ### Cleaning rules -distclean: clean - $(RM) configure +autoclean: + $(RM) configure aclocal.m4 config.status config.cache \ + install-sh missing t/Makefile.auto t/Makefile.auto.in \ + config.log config.mak.autogen config.mak.append + $(RM) -r autom4te.cache +.PHONY: autoclean + +distclean: clean autoclean $(RM) po/git.pot clean: @@ -2356,8 +2378,6 @@ clean: $(RM) -r bin-wrappers $(RM) -r $(dep_dirs) $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo common-cmds.h $(ETAGS_TARGET) tags cscope* - $(RM) -r autom4te.cache - $(RM) config.log config.mak.autogen config.mak.append config.status config.cache $(RM) -r $(GIT_TARNAME) .doc-tmp-dir $(RM) $(GIT_TARNAME).tar.gz git-core_$(GIT_VERSION)-*.tar.gz $(RM) $(htmldocs).tar.gz $(manpages).tar.gz diff --git a/aclocal.m4 b/acinclude.m4 similarity index 100% rename from aclocal.m4 rename to acinclude.m4 diff --git a/configure.ac b/configure.ac index 048a1d4..581a005 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,10 @@ # -*- Autoconf -*- # Process this file with autoconf to produce a configure script. -AC_PREREQ(2.59) -AC_INIT([git], [@@GIT_VERSION@@], [g...@vger.kernel.org]) +AC_PREREQ([2.62]) +AC_INIT([git], + m4_esyscmd([cat GIT-VERSION-FILE | sed 's/.*= *//' | tr -d '\n']), + [g...@vger.kernel.org]) AC_CONFIG_SRCDIR([git.c]) @@ -12,6 +14,8 @@ config_in=config.mak.in echo "# ${config_append}. Generated by configure." > "${config_append}" +AC_CONFIG_AUX_DIR([.]) +AM_INIT_AUTOMAKE([-Wall -Werror -Wno-override -Wno-portability]) ## Definitions of macros # GIT_CONF_APPEND_LINE(LINE) @@ -999,7 +1003,10 @@ AC_SUBST(PTHREAD_LIBS) AC_SUBST(NO_PTHREADS) ## Output files -AC_CONFIG_FILES(["${config_file}":"${config_in}":"${config_append}"]) +AC_CONFIG_FILES([ + "${config_file}":"${config_in}":"${config_append}" + t/Makefile.auto +]) AC_OUTPUT diff --git a/t/.gitignore b/t/.gitignore index 4e731dc..cd6f5a7 100644 --- a/t/.gitignore +++ b/t/.gitignore @@ -1,3 +1,7 @@ /trash directory* /test-results /.prove +/Makefile.auto.in +/Makefile.auto +/*.trs +/*.log diff --git a/t/Makefile b/t/Makefile index 9046ec9..1e6e2da 100644 --- a/t/Makefile +++ b/t/Makefile @@ -6,6 +6,10 @@ -include ../config.mak.autogen -include ../config.mak +ifeq ($(use_automake_test_harness),yes) +include Makefile.auto +else + #GIT_TEST_OPTS=--verbose --debug SHELL_PATH ?= $(SHELL) PERL_PATH ?= /usr/bin/perl @@ -112,3 +116,5 @@ smoke_report: smoke | grep -v ^Redirecting .PHONY: pre-clean $(T) aggregate-results clean valgrind smoke smoke_report + +endif diff --git a/t/Makefile.auto.am b/t/Makefile.auto.am new file mode 100644 index 0000000..41f0fec --- /dev/null +++ b/t/Makefile.auto.am @@ -0,0 +1,591 @@ +## -*- Makefile -*- + +AUTOMAKE_OPTIONS = color-tests parallel-tests no-dist + +TEST_EXTENSIONS = .sh +SH_LOG_DRIVER = $(PERL_PATH) ./tap-driver +## For compatibility with TAP::Harness. The test scripts use this +## variable to determine whether they are running directory or +## through an harness, and to adapt their behaviour accordingly. +AM_TESTS_ENVIRONMENT = HARNESS_ACTIVE=yes + +TESTS = $(all_TESTS) # Can be overridden at make time. + +## Nullify some targets that we don't need. +install: +uninstall: +install-strip: +install-exec: +install-data: +## For consistency with the non-automake Makefile. +all: check check-list-of-tests +.PHONY: install uninstall all + +## Temporary files used in the `check-list-of-tests' target below. +am__tmk = tests-in-makefile-list.tmp +am__tfs = tests-on-filesystem-list.tmp +am__tdf = diff-in-tests-lists.tmp + +## Check that the list of tests given in the Makefile is equal to the +## list of all test scripts in the Automake testsuite. +.PHONY: check-list-of-tests +check-list-of-tests: +## Prefer unified diffs over plain diffs, for readability. + @if diff -u /dev/null /dev/null >/dev/null 2>&1; then \ + diff='diff -u'; \ + else \ + diff='diff'; \ + fi; \ + LC_ALL=C; export LC_ALL; \ +## List of tests in Makefile. + for t in $(all_TESTS); do \ + echo "$$t"; \ + done | sort >$(am__tmk); \ +## List of tests on filesystem. + for t in t[0-9][0-9][0-9][0-9]-*.sh; do \ + echo "$$t"; \ + done | sort >$(am__tfs); \ +## Compare them. + if $$diff $(am__tmk) $(am__tfs) >$(am__tdf); then \ + result=0; \ + else \ + echo 'List of tests in Makefile and on filesystem differ' >&2; \ + echo "+ $$diff in-makefile on-filesystem" >&2; \ + cat $(am__tdf) >&2; \ + result=1; \ + fi; \ + rm -f $(am__tmk) $(am__tfs) $(am__tdf); \ + exit $$result; + +clean-local: + rm -f $(am__tmk) $(am__tfs) $(am__tdf) + +all_TESTS = \ + t0000-basic.sh \ + t0001-init.sh \ + t0002-gitfile.sh \ + t0003-attributes.sh \ + t0004-unwritable.sh \ + t0005-signals.sh \ + t0006-date.sh \ + t0010-racy-git.sh \ + t0020-crlf.sh \ + t0021-conversion.sh \ + t0022-crlf-rename.sh \ + t0023-crlf-am.sh \ + t0024-crlf-archive.sh \ + t0025-crlf-auto.sh \ + t0026-eol-config.sh \ + t0030-stripspace.sh \ + t0040-parse-options.sh \ + t0050-filesystem.sh \ + t0055-beyond-symlinks.sh \ + t0060-path-utils.sh \ + t0061-run-command.sh \ + t0070-fundamental.sh \ + t0080-vcs-svn.sh \ + t0081-line-buffer.sh \ + t0100-previous.sh \ + t0101-at-syntax.sh \ + t0201-gettext-fallbacks.sh \ + t1000-read-tree-m-3way.sh \ + t1001-read-tree-m-2way.sh \ + t1002-read-tree-m-u-2way.sh \ + t1003-read-tree-prefix.sh \ + t1004-read-tree-m-u-wf.sh \ + t1005-read-tree-reset.sh \ + t1006-cat-file.sh \ + t1007-hash-object.sh \ + t1008-read-tree-overlay.sh \ + t1009-read-tree-new-index.sh \ + t1010-mktree.sh \ + t1011-read-tree-sparse-checkout.sh \ + t1012-read-tree-df.sh \ + t1020-subdirectory.sh \ + t1021-rerere-in-workdir.sh \ + t1050-large.sh \ + t1100-commit-tree-options.sh \ + t1200-tutorial.sh \ + t1300-repo-config.sh \ + t1301-shared-repo.sh \ + t1302-repo-version.sh \ + t1303-wacky-config.sh \ + t1304-default-acl.sh \ + t1400-update-ref.sh \ + t1401-symbolic-ref.sh \ + t1402-check-ref-format.sh \ + t1410-reflog.sh \ + t1411-reflog-show.sh \ + t1412-reflog-loop.sh \ + t1420-lost-found.sh \ + t1450-fsck.sh \ + t1500-rev-parse.sh \ + t1501-worktree.sh \ + t1502-rev-parse-parseopt.sh \ + t1503-rev-parse-verify.sh \ + t1504-ceiling-dirs.sh \ + t1505-rev-parse-last.sh \ + t1506-rev-parse-diagnosis.sh \ + t1507-rev-parse-upstream.sh \ + t1508-at-combinations.sh \ + t1509-root-worktree.sh \ + t1510-repo-setup.sh \ + t1511-rev-parse-caret.sh \ + t2000-checkout-cache-clash.sh \ + t2001-checkout-cache-clash.sh \ + t2002-checkout-cache-u.sh \ + t2003-checkout-cache-mkdir.sh \ + t2004-checkout-cache-temp.sh \ + t2005-checkout-index-symlinks.sh \ + t2006-checkout-index-basic.sh \ + t2007-checkout-symlink.sh \ + t2008-checkout-subdir.sh \ + t2009-checkout-statinfo.sh \ + t2010-checkout-ambiguous.sh \ + t2011-checkout-invalid-head.sh \ + t2012-checkout-last.sh \ + t2013-checkout-submodule.sh \ + t2014-switch.sh \ + t2015-checkout-unborn.sh \ + t2016-checkout-patch.sh \ + t2017-checkout-orphan.sh \ + t2018-checkout-branch.sh \ + t2019-checkout-ambiguous-ref.sh \ + t2020-checkout-detach.sh \ + t2021-checkout-overwrite.sh \ + t2030-unresolve-info.sh \ + t2050-git-dir-relative.sh \ + t2100-update-cache-badpath.sh \ + t2101-update-index-reupdate.sh \ + t2102-update-index-symlinks.sh \ + t2103-update-index-ignore-missing.sh \ + t2104-update-index-skip-worktree.sh \ + t2105-update-index-gitfile.sh \ + t2106-update-index-assume-unchanged.sh \ + t2107-update-index-basic.sh \ + t2200-add-update.sh \ + t2201-add-update-typechange.sh \ + t2202-add-addremove.sh \ + t2203-add-intent.sh \ + t2204-add-ignored.sh \ + t2300-cd-to-toplevel.sh \ + t3000-ls-files-others.sh \ + t3001-ls-files-others-exclude.sh \ + t3002-ls-files-dashpath.sh \ + t3003-ls-files-exclude.sh \ + t3004-ls-files-basic.sh \ + t3010-ls-files-killed-modified.sh \ + t3020-ls-files-error-unmatch.sh \ + t3030-merge-recursive.sh \ + t3031-merge-criscross.sh \ + t3032-merge-recursive-options.sh \ + t3040-subprojects-basic.sh \ + t3050-subprojects-fetch.sh \ + t3060-ls-files-with-tree.sh \ + t3100-ls-tree-restrict.sh \ + t3101-ls-tree-dirname.sh \ + t3102-ls-tree-wildcards.sh \ + t3200-branch.sh \ + t3201-branch-contains.sh \ + t3202-show-branch-octopus.sh \ + t3203-branch-output.sh \ + t3210-pack-refs.sh \ + t3300-funny-names.sh \ + t3301-notes.sh \ + t3302-notes-index-expensive.sh \ + t3303-notes-subtrees.sh \ + t3304-notes-mixed.sh \ + t3305-notes-fanout.sh \ + t3306-notes-prune.sh \ + t3307-notes-man.sh \ + t3308-notes-merge.sh \ + t3309-notes-merge-auto-resolve.sh \ + t3310-notes-merge-manual-resolve.sh \ + t3311-notes-merge-fanout.sh \ + t3400-rebase.sh \ + t3401-rebase-partial.sh \ + t3402-rebase-merge.sh \ + t3403-rebase-skip.sh \ + t3404-rebase-interactive.sh \ + t3405-rebase-malformed.sh \ + t3406-rebase-message.sh \ + t3407-rebase-abort.sh \ + t3408-rebase-multi-line.sh \ + t3409-rebase-preserve-merges.sh \ + t3410-rebase-preserve-dropped-merges.sh \ + t3411-rebase-preserve-around-merges.sh \ + t3412-rebase-root.sh \ + t3413-rebase-hook.sh \ + t3414-rebase-preserve-onto.sh \ + t3415-rebase-autosquash.sh \ + t3416-rebase-onto-threedots.sh \ + t3417-rebase-whitespace-fix.sh \ + t3418-rebase-continue.sh \ + t3419-rebase-patch-id.sh \ + t3500-cherry.sh \ + t3501-revert-cherry-pick.sh \ + t3502-cherry-pick-merge.sh \ + t3503-cherry-pick-root.sh \ + t3504-cherry-pick-rerere.sh \ + t3505-cherry-pick-empty.sh \ + t3506-cherry-pick-ff.sh \ + t3507-cherry-pick-conflict.sh \ + t3508-cherry-pick-many-commits.sh \ + t3509-cherry-pick-merge-df.sh \ + t3600-rm.sh \ + t3700-add.sh \ + t3701-add-interactive.sh \ + t3702-add-edit.sh \ + t3703-add-magic-pathspec.sh \ + t3800-mktag.sh \ + t3900-i18n-commit.sh \ + t3901-i18n-patch.sh \ + t3902-quoted.sh \ + t3903-stash.sh \ + t3904-stash-patch.sh \ + t4000-diff-format.sh \ + t4001-diff-rename.sh \ + t4002-diff-basic.sh \ + t4003-diff-rename-1.sh \ + t4004-diff-rename-symlink.sh \ + t4005-diff-rename-2.sh \ + t4006-diff-mode.sh \ + t4007-rename-3.sh \ + t4008-diff-break-rewrite.sh \ + t4009-diff-rename-4.sh \ + t4010-diff-pathspec.sh \ + t4011-diff-symlink.sh \ + t4012-diff-binary.sh \ + t4013-diff-various.sh \ + t4014-format-patch.sh \ + t4015-diff-whitespace.sh \ + t4016-diff-quote.sh \ + t4017-diff-retval.sh \ + t4018-diff-funcname.sh \ + t4019-diff-wserror.sh \ + t4020-diff-external.sh \ + t4021-format-patch-numbered.sh \ + t4022-diff-rewrite.sh \ + t4023-diff-rename-typechange.sh \ + t4024-diff-optimize-common.sh \ + t4025-hunk-header.sh \ + t4026-color.sh \ + t4027-diff-submodule.sh \ + t4028-format-patch-mime-headers.sh \ + t4029-diff-trailing-space.sh \ + t4030-diff-textconv.sh \ + t4031-diff-rewrite-binary.sh \ + t4032-diff-inter-hunk-context.sh \ + t4033-diff-patience.sh \ + t4034-diff-words.sh \ + t4035-diff-quiet.sh \ + t4036-format-patch-signer-mime.sh \ + t4037-diff-r-t-dirs.sh \ + t4038-diff-combined.sh \ + t4039-diff-assume-unchanged.sh \ + t4040-whitespace-status.sh \ + t4041-diff-submodule-option.sh \ + t4042-diff-textconv-caching.sh \ + t4043-diff-rename-binary.sh \ + t4044-diff-index-unique-abbrev.sh \ + t4045-diff-relative.sh \ + t4046-diff-unmerged.sh \ + t4047-diff-dirstat.sh \ + t4100-apply-stat.sh \ + t4101-apply-nonl.sh \ + t4102-apply-rename.sh \ + t4103-apply-binary.sh \ + t4104-apply-boundary.sh \ + t4105-apply-fuzz.sh \ + t4106-apply-stdin.sh \ + t4107-apply-ignore-whitespace.sh \ + t4109-apply-multifrag.sh \ + t4110-apply-scan.sh \ + t4111-apply-subdir.sh \ + t4112-apply-renames.sh \ + t4113-apply-ending.sh \ + t4114-apply-typechange.sh \ + t4115-apply-symlink.sh \ + t4116-apply-reverse.sh \ + t4117-apply-reject.sh \ + t4118-apply-empty-context.sh \ + t4119-apply-config.sh \ + t4120-apply-popt.sh \ + t4121-apply-diffs.sh \ + t4122-apply-symlink-inside.sh \ + t4123-apply-shrink.sh \ + t4124-apply-ws-rule.sh \ + t4125-apply-ws-fuzz.sh \ + t4126-apply-empty.sh \ + t4127-apply-same-fn.sh \ + t4128-apply-root.sh \ + t4129-apply-samemode.sh \ + t4130-apply-criss-cross-rename.sh \ + t4131-apply-fake-ancestor.sh \ + t4132-apply-removal.sh \ + t4133-apply-filenames.sh \ + t4134-apply-submodule.sh \ + t4135-apply-weird-filenames.sh \ + t4150-am.sh \ + t4151-am-abort.sh \ + t4152-am-subjects.sh \ + t4200-rerere.sh \ + t4201-shortlog.sh \ + t4202-log.sh \ + t4203-mailmap.sh \ + t4204-patch-id.sh \ + t4205-log-pretty-formats.sh \ + t4206-log-follow-harder-copies.sh \ + t4207-log-decoration-colors.sh \ + t4208-log-magic-pathspec.sh \ + t4252-am-options.sh \ + t4253-am-keep-cr-dos.sh \ + t4300-merge-tree.sh \ + t5000-tar-tree.sh \ + t5001-archive-attr.sh \ + t5100-mailinfo.sh \ + t5150-request-pull.sh \ + t5300-pack-object.sh \ + t5301-sliding-window.sh \ + t5302-pack-index.sh \ + t5303-pack-corruption-resilience.sh \ + t5304-prune.sh \ + t5305-include-tag.sh \ + t5306-pack-nobase.sh \ + t5307-pack-missing-commit.sh \ + t5400-send-pack.sh \ + t5401-update-hooks.sh \ + t5402-post-merge-hook.sh \ + t5403-post-checkout-hook.sh \ + t5404-tracking-branches.sh \ + t5405-send-pack-rewind.sh \ + t5406-remote-rejects.sh \ + t5407-post-rewrite-hook.sh \ + t5500-fetch-pack.sh \ + t5501-fetch-push-alternates.sh \ + t5502-quickfetch.sh \ + t5503-tagfollow.sh \ + t5505-remote.sh \ + t5506-remote-groups.sh \ + t5510-fetch.sh \ + t5511-refspec.sh \ + t5512-ls-remote.sh \ + t5513-fetch-track.sh \ + t5514-fetch-multiple.sh \ + t5515-fetch-merge-logic.sh \ + t5516-fetch-push.sh \ + t5517-push-mirror.sh \ + t5518-fetch-exit-status.sh \ + t5519-push-alternates.sh \ + t5520-pull.sh \ + t5521-pull-options.sh \ + t5522-pull-symlink.sh \ + t5523-push-upstream.sh \ + t5524-pull-msg.sh \ + t5525-fetch-tagopt.sh \ + t5526-fetch-submodules.sh \ + t5530-upload-pack-error.sh \ + t5531-deep-submodule-push.sh \ + t5532-fetch-proxy.sh \ + t5540-http-push.sh \ + t5541-http-push.sh \ + t5550-http-fetch.sh \ + t5551-http-fetch.sh \ + t5560-http-backend-noserver.sh \ + t5561-http-backend.sh \ + t5600-clone-fail-cleanup.sh \ + t5601-clone.sh \ + t5602-clone-remote-exec.sh \ + t5700-clone-reference.sh \ + t5701-clone-local.sh \ + t5702-clone-options.sh \ + t5704-bundle.sh \ + t5705-clone-2gb.sh \ + t5706-clone-branch.sh \ + t5710-info-alternate.sh \ + t5800-remote-helpers.sh \ + t6000-rev-list-misc.sh \ + t6001-rev-list-graft.sh \ + t6002-rev-list-bisect.sh \ + t6003-rev-list-topo-order.sh \ + t6004-rev-list-path-optim.sh \ + t6005-rev-list-count.sh \ + t6006-rev-list-format.sh \ + t6007-rev-list-cherry-pick-file.sh \ + t6008-rev-list-submodule.sh \ + t6009-rev-list-parent.sh \ + t6010-merge-base.sh \ + t6011-rev-list-with-bad-commit.sh \ + t6012-rev-list-simplify.sh \ + t6013-rev-list-reverse-parents.sh \ + t6014-rev-list-all.sh \ + t6015-rev-list-show-all-parents.sh \ + t6016-rev-list-graph-simplify-history.sh \ + t6017-rev-list-stdin.sh \ + t6018-rev-list-glob.sh \ + t6019-rev-list-ancestry-path.sh \ + t6020-merge-df.sh \ + t6021-merge-criss-cross.sh \ + t6022-merge-rename.sh \ + t6023-merge-file.sh \ + t6024-recursive-merge.sh \ + t6025-merge-symlinks.sh \ + t6026-merge-attr.sh \ + t6027-merge-binary.sh \ + t6028-merge-up-to-date.sh \ + t6029-merge-subtree.sh \ + t6030-bisect-porcelain.sh \ + t6031-merge-recursive.sh \ + t6032-merge-large-rename.sh \ + t6033-merge-crlf.sh \ + t6034-merge-rename-nocruft.sh \ + t6035-merge-dir-to-symlink.sh \ + t6036-recursive-corner-cases.sh \ + t6037-merge-ours-theirs.sh \ + t6038-merge-text-auto.sh \ + t6040-tracking-info.sh \ + t6050-replace.sh \ + t6060-merge-index.sh \ + t6101-rev-parse-parents.sh \ + t6110-rev-list-sparse.sh \ + t6120-describe.sh \ + t6200-fmt-merge-msg.sh \ + t6300-for-each-ref.sh \ + t6500-gc.sh \ + t7001-mv.sh \ + t7003-filter-branch.sh \ + t7004-tag.sh \ + t7005-editor.sh \ + t7006-pager.sh \ + t7007-show.sh \ + t7008-grep-binary.sh \ + t7010-setup.sh \ + t7011-skip-worktree-reading.sh \ + t7012-skip-worktree-writing.sh \ + t7060-wtstatus.sh \ + t7101-reset.sh \ + t7102-reset.sh \ + t7103-reset-bare.sh \ + t7104-reset.sh \ + t7105-reset-patch.sh \ + t7110-reset-merge.sh \ + t7111-reset-table.sh \ + t7201-co.sh \ + t7300-clean.sh \ + t7400-submodule-basic.sh \ + t7401-submodule-summary.sh \ + t7402-submodule-rebase.sh \ + t7403-submodule-sync.sh \ + t7405-submodule-merge.sh \ + t7406-submodule-update.sh \ + t7407-submodule-foreach.sh \ + t7408-submodule-reference.sh \ + t7500-commit.sh \ + t7501-commit.sh \ + t7502-commit.sh \ + t7503-pre-commit-hook.sh \ + t7504-commit-msg-hook.sh \ + t7505-prepare-commit-msg-hook.sh \ + t7506-status-submodule.sh \ + t7507-commit-verbose.sh \ + t7508-status.sh \ + t7509-commit.sh \ + t7600-merge.sh \ + t7601-merge-pull-config.sh \ + t7602-merge-octopus-many.sh \ + t7603-merge-reduce-heads.sh \ + t7604-merge-custom-message.sh \ + t7605-merge-resolve.sh \ + t7606-merge-custom.sh \ + t7607-merge-overwrite.sh \ + t7608-merge-messages.sh \ + t7609-merge-co-error-msgs.sh \ + t7610-mergetool.sh \ + t7611-merge-abort.sh \ + t7700-repack.sh \ + t7701-repack-unpack-unreachable.sh \ + t7800-difftool.sh \ + t7810-grep.sh \ + t7811-grep-open.sh \ + t8001-annotate.sh \ + t8002-blame.sh \ + t8003-blame-corner-cases.sh \ + t8004-blame-with-conflicts.sh \ + t8005-blame-i18n.sh \ + t8006-blame-textconv.sh \ + t8007-cat-file-textconv.sh \ + t8008-blame-formats.sh \ + t9001-send-email.sh \ + t9010-svn-fe.sh \ + t9100-git-svn-basic.sh \ + t9101-git-svn-props.sh \ + t9102-git-svn-deep-rmdir.sh \ + t9103-git-svn-tracked-directory-removed.sh \ + t9104-git-svn-follow-parent.sh \ + t9105-git-svn-commit-diff.sh \ + t9106-git-svn-commit-diff-clobber.sh \ + t9107-git-svn-migrate.sh \ + t9108-git-svn-glob.sh \ + t9109-git-svn-multi-glob.sh \ + t9110-git-svn-use-svm-props.sh \ + t9111-git-svn-use-svnsync-props.sh \ + t9112-git-svn-md5less-file.sh \ + t9113-git-svn-dcommit-new-file.sh \ + t9114-git-svn-dcommit-merge.sh \ + t9115-git-svn-dcommit-funky-renames.sh \ + t9116-git-svn-log.sh \ + t9117-git-svn-init-clone.sh \ + t9118-git-svn-funky-branch-names.sh \ + t9119-git-svn-info.sh \ + t9120-git-svn-clone-with-percent-escapes.sh \ + t9121-git-svn-fetch-renamed-dir.sh \ + t9122-git-svn-author.sh \ + t9123-git-svn-rebuild-with-rewriteroot.sh \ + t9124-git-svn-dcommit-auto-props.sh \ + t9125-git-svn-multi-glob-branch-names.sh \ + t9126-git-svn-follow-deleted-readded-directory.sh \ + t9127-git-svn-partial-rebuild.sh \ + t9128-git-svn-cmd-branch.sh \ + t9129-git-svn-i18n-commitencoding.sh \ + t9130-git-svn-authors-file.sh \ + t9131-git-svn-empty-symlink.sh \ + t9132-git-svn-broken-symlink.sh \ + t9133-git-svn-nested-git-repo.sh \ + t9134-git-svn-ignore-paths.sh \ + t9135-git-svn-moved-branch-empty-file.sh \ + t9136-git-svn-recreated-branch-empty-file.sh \ + t9137-git-svn-dcommit-clobber-series.sh \ + t9138-git-svn-authors-prog.sh \ + t9139-git-svn-non-utf8-commitencoding.sh \ + t9140-git-svn-reset.sh \ + t9141-git-svn-multiple-branches.sh \ + t9142-git-svn-shallow-clone.sh \ + t9143-git-svn-gc.sh \ + t9144-git-svn-old-rev_map.sh \ + t9145-git-svn-master-branch.sh \ + t9146-git-svn-empty-dirs.sh \ + t9150-svk-mergetickets.sh \ + t9151-svn-mergeinfo.sh \ + t9152-svn-empty-dirs-after-gc.sh \ + t9153-git-svn-rewrite-uuid.sh \ + t9154-git-svn-fancy-glob.sh \ + t9155-git-svn-fetch-deleted-tag.sh \ + t9156-git-svn-fetch-deleted-tag-2.sh \ + t9157-git-svn-fetch-merge.sh \ + t9158-git-svn-mergeinfo.sh \ + t9159-git-svn-no-parent-mergeinfo.sh \ + t9200-git-cvsexportcommit.sh \ + t9300-fast-import.sh \ + t9301-fast-import-notes.sh \ + t9350-fast-export.sh \ + t9400-git-cvsserver-server.sh \ + t9401-git-cvsserver-crlf.sh \ + t9500-gitweb-standalone-no-errors.sh \ + t9501-gitweb-standalone-http-status.sh \ + t9502-gitweb-standalone-parse-output.sh \ + t9600-cvsimport.sh \ + t9601-cvsimport-vendor-branch.sh \ + t9602-cvsimport-branches-tags.sh \ + t9603-cvsimport-patchsets.sh \ + t9700-perl-git.sh \ + t9800-git-p4.sh diff --git a/t/tap-driver b/t/tap-driver new file mode 100755 index 0000000..2393346 --- /dev/null +++ b/t/tap-driver @@ -0,0 +1,451 @@ +#! /usr/bin/env perl +# Temporary/experimental TAP test driver for Automake. +# TODO: should be rewritten portably (e.g., in awk or shell). + +# ---------------------------------- # +# Imports, static data, and setup. # +# ---------------------------------- # + +use warnings FATAL => 'all'; +use strict; +use Getopt::Long (); +use TAP::Parser; + +my $ME = "tap-driver"; + +my $USAGE = <<'END'; +Usage: + tap-driver --test-name=NAME --log-file=PATH --trs-file=PATH + [--expect-failure={yes|no}] [--color-tests={yes|no}] + [--enable-hard-errors={yes|no}] [--ignore-exit] + [--diagnostic-string=STRING] [--merge|--no-merge] + [--comments|--no-comments] [--] TEST-COMMAND +The `--test-name' and `--log-file' options are mandatory. +END + +my $HELP = "$ME: TAP-aware test driver for Automake testsuite harness." . + "\n" . $USAGE; + +my $VERSION = '(experimental version)'; + +# Keep this in sync with `lib/am/check.am:$(am__tty_colors)'. +my %COLOR = ( + red => "\e[0;31m", + grn => "\e[0;32m", + lgn => "\e[1;32m", + blu => "\e[1;34m", + mgn => "\e[0;35m", + brg => "\e[1m", + std => "\e[m", +); + +# ------------------- # +# Global variables. # +# ------------------- # + +my $testno = 0; # Number of test results seen so far. +my $plan_seen = 0; # Whether the TAP plan has been seen or not. +my $parser; # TAP parser object (will be initialized later). + +# When true, it means that the rest of the input stream cannot +# contain any further TAP results. +my $tap_stopped = 0; + +# ----------------- # +# Option parsing. # +# ----------------- # + +my %cfg = ( + "color-tests" => 0, + "expect-failure" => 0, + "enable-hard-errors" => 1, + "merge" => 0, + "comments" => 0, + "ignore-exit" => 0, +); + +my $test_script_name = undef; +my $log_file = undef; +my $trs_file = undef; +my $diag_string = "#"; + +Getopt::Long::GetOptions ( + 'help' => sub { print $HELP; exit 0; }, + 'version' => sub { print "$ME $VERSION\n"; exit 0; }, + 'test-name=s' => \$test_script_name, + 'log-file=s' => \$log_file, + 'trs-file=s' => \$trs_file, + 'color-tests=s' => \&bool_opt, + 'expect-failure=s' => \&bool_opt, + 'enable-hard-errors=s' => \&bool_opt, + 'diagnostic-string=s' => \$diag_string, + 'comments' => sub { $cfg{"comments"} = 1; }, + 'no-comments' => sub { $cfg{"comments"} = 0; }, + 'merge' => sub { $cfg{"merge"} = 1; }, + 'no-merge' => sub { $cfg{"merge"} = 0; }, + 'ignore-exit' => sub { $cfg{"ignore-exit"} = 1; }, + ) or exit 1; + +# ------------- # +# Prototypes. # +# ------------- # + +sub add_test_result ($); +sub bool_opt ($$); +sub colored ($$); +sub copy_in_global_log (); +sub decorate_result ($); +sub extract_tap_comment ($); +sub finish (); +sub get_global_test_result (); +sub get_test_exit_message (); +sub get_test_results (); +sub handle_tap_bailout ($); +sub handle_tap_plan ($); +sub handle_tap_test ($); +sub main (@); +sub must_recheck (); +sub report ($;$); +sub start (@); +sub stringify_test_result ($); +sub testsuite_error ($); +sub write_test_results (); +sub yn ($); + +# -------------- # +# Subroutines. # +# -------------- # + +sub bool_opt ($$) +{ + my ($opt, $val) = @_; + if ($val =~ /^(?:y|yes)\z/i) + { + $cfg{$opt} = 1; + } + elsif ($val =~ /^(?:n|no)\z/i) + { + $cfg{$opt} = 0; + } + else + { + die "invalid argument '$val' for option '$opt'\n"; + } +} + +# Convert a boolean to a "yes"/"no" string. +sub yn ($) +{ + my $bool = shift; + return $bool ? "yes" : "no"; +} + +TEST_RESULTS : +{ + my (@test_results, %test_results); + + sub add_test_result ($) + { + my $res = shift; + push @test_results, $res; + $test_results{$res} = 1; + } + + sub get_test_results () + { + return @test_results; + } + + # Whether the test script should be re-run by "make recheck". + sub must_recheck () + { + return grep { !/^(?:XFAIL|PASS|SKIP)$/ } (keys %test_results); + } + + # Whether the content of the log file associated to this test should + # be copied into the "global" test-suite.log. + sub copy_in_global_log () + { + return grep { not $_ eq "PASS" } (keys %test_results); + } + + # FIXME: this can certainly be improved ... + sub get_global_test_result () + { + my @results = keys %test_results; + return "ERROR" if exists $test_results{"ERROR"}; + return "SKIP" if @results == 1 && $results[0] eq "SKIP"; + return "FAIL" if exists $test_results{"FAIL"}; + return "FAIL" if exists $test_results{"XPASS"}; + return "PASS"; + } + +} + +sub write_test_results () +{ + open RES, ">", $trs_file or die "opening $trs_file: $!\n"; + print RES ":global-test-result: " . get_global_test_result . "\n"; + print RES ":recheck: " . yn (must_recheck) . "\n"; + print RES ":copy-in-global-log: " . yn (copy_in_global_log) . "\n"; + foreach my $result (get_test_results) + { + print RES ":test-result: $result\n"; + } + close RES or die "closing $trs_file: $!\n"; +} + +sub start (@) +{ + # Redirect stderr and stdout to a temporary log file. Save the + # original stdout stream, since we need it to print testsuite + # progress output. + open LOG, ">", $log_file or die "opening $log_file: $!\n"; + open OLDOUT, ">&STDOUT" or die "duplicating stdout: $!\n"; + open STDOUT, ">&LOG" or die "redirecting stdout: $!\n"; + open STDERR, ">&LOG" or die "redirecting stderr: $!\n"; + $parser = TAP::Parser->new ({ exec => \@_, merge => $cfg{merge} }); + $parser->ignore_exit(1) if $cfg{"ignore-exit"}; +} + +sub get_test_exit_message () +{ + # Flush all the remaining TAP stream, so that we can obtain the + # exit status of the TAP producer. + do {} while defined $parser->next; + my $wstatus = $parser->wait; + # Return an undefined value if the producer exited with success. + return unless $wstatus; + # Otherwise, determine whether it exited with error or was terminated + # by a signal. + use POSIX qw (WIFEXITED WEXITSTATUS WIFSIGNALED WTERMSIG); + if (WIFEXITED ($wstatus)) + { + return sprintf "exited with status %d", WEXITSTATUS ($wstatus); + } + elsif (WIFSIGNALED ($wstatus)) + { + return sprintf "terminated by signal %d", WTERMSIG ($wstatus); + } + else + { + return "terminated abnormally"; + } +} + +sub finish () +{ + if (!$cfg{"ignore-exit"} and my $msg = get_test_exit_message) + { + testsuite_error $msg; + } + write_test_results; + close LOG or die "closing $log_file: $!\n"; + exit 0; +} + +sub stringify_test_result ($) +{ + my $result = shift; + my $PASS = $cfg{"expect-failure"} ? "XPASS": "PASS"; + my $FAIL = $cfg{"expect-failure"} ? "XFAIL": "FAIL"; + if ($result->is_unplanned || $result->number != $testno || $tap_stopped) + { + return "ERROR"; + } + elsif (!$result->directive) + { + return $result->is_ok ? $PASS: $FAIL; + } + elsif ($result->has_todo) + { + return $result->is_actual_ok ? "XPASS" : "XFAIL"; + } + elsif ($result->has_skip) + { + return $result->is_ok ? "SKIP" : $FAIL; + } + die "INTERNAL ERROR"; # NOTREACHED +} + +sub colored ($$) +{ + my ($color_name, $text) = @_; + return $COLOR{$color_name} . $text . $COLOR{'std'}; +} + +sub decorate_result ($) +{ + my $result = shift; + return $result unless $cfg{"color-tests"}; + my %color_for_result = + ( + "ERROR" => 'mgn', + "PASS" => 'grn', + "XPASS" => 'red', + "FAIL" => 'red', + "XFAIL" => 'lgn', + "SKIP" => 'blu', + ); + if (my $color = $color_for_result{$result}) + { + return colored ($color, $result); + } + else + { + return $result; # Don't colorize unknown stuff. + } +} + +sub report ($;$) +{ + my ($msg, $result, $explanation) = (undef, @_); + if ($result =~ /^(?:X?(?:PASS|FAIL)|SKIP|ERROR)/) + { + $msg = ": $test_script_name"; + add_test_result $result; + } + elsif ($result eq "#") + { + $msg = " $test_script_name:"; + } + else + { + die "INTERNAL ERROR"; # NOTREACHED + } + $msg .= " $explanation" if defined $explanation; + $msg .= "\n"; + # Output on console might be colorized. + print OLDOUT decorate_result ($result) . $msg; + # Log the result in the log file too, to help debugging (this is + # especially true when said result is a TAP error or "Bail out!"). + print $result . $msg; +} + +sub testsuite_error ($) +{ + report "ERROR", "- $_[0]"; +} + +sub handle_tap_test ($) +{ + $testno++; + my $test = shift; + + my $test_result = stringify_test_result $test; + my $string = $test->number; + + if (my $description = $test->description) + { + $string .= " $description"; + } + + if ($tap_stopped) + { + $string .= " # AFTER LATE PLAN"; + } + elsif ($test->is_unplanned) + { + $string .= " # UNPLANNED"; + } + elsif ($test->number != $testno) + { + $string .= " # OUT-OF-ORDER (expecting $testno)"; + } + elsif (my $directive = $test->directive) + { + $string .= " # $directive"; + if (my $explanation = $test->explanation) + { + $string .= " $explanation"; + } + } + + report $test_result, $string; +} + +sub handle_tap_plan ($) +{ + my $plan = shift; + # Only one plan per stream is acceptable. + testsuite_error "multiple test plans" if $plan_seen; + $plan_seen = 1; + # TAP plan must come either before or after *all* the TAP results. + # So, if we find it after having already seen at least one TAP result, + # set a flag signaling that no more TAP results are acceptable. + $tap_stopped = 1 if $testno >= 1; + # Nothing more to do, unless the plan contains a SKIP directive. + return + if not defined $plan->directive && length ($plan->directive) > 0; + my $explanation = $plan->explanation ? + "- " . $plan->explanation : undef; + report "SKIP", $explanation; + finish; +} + +sub handle_tap_bailout ($) +{ + my ($bailout, $msg) = ($_[0], "Bail out!"); + $msg .= " " . $bailout->explanation if $bailout->explanation; + testsuite_error $msg; + finish; +} + +sub extract_tap_comment ($) +{ + local $_ = shift; + if (/^\Q$diag_string\E(.*)$/o) + { + (my $comment = $1) =~ s/(?:^\s*|\s*$)//g; + return $comment; + } + return ""; +} + +sub main (@) +{ + start @_; + + while (defined (my $cur = $parser->next)) + { + # Verbatim copy any input line into the log file. + print $cur->raw . "\n"; + if ($cur->is_plan) + { + handle_tap_plan ($cur); + } + elsif ($cur->is_test) + { + handle_tap_test ($cur); + } + elsif ($cur->is_bailout) + { + handle_tap_bailout ($cur); + } + elsif ($cfg{comments}) + { + my $comment = extract_tap_comment ($cur->raw); + report "#", "$comment" if length $comment; + } + } + if (!$plan_seen) + { + testsuite_error "missing test plan"; + } + elsif ($parser->tests_planned != $parser->tests_run) + { + my ($planned, $run) = ($parser->tests_planned, $parser->tests_run); + my $bad_amount = $run > $planned ? "many" : "few"; + testsuite_error (sprintf "too %s tests run (expected %d, got %d)", + $bad_amount, $planned, $run); + } + finish; +} + +# ----------- # +# Main code. # +# ----------- # + +main @ARGV; + +# vim: ft=perl ts=4 sw=4 et diff --git a/t/test-lib.sh b/t/test-lib.sh index df25f17..36a082c 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -854,7 +854,11 @@ test_done () { say "1..$test_count" fi - exit 1 ;; + if test -z "$HARNESS_ACTIVE"; then + exit 1 + else + exit 0 + fi ;; esac } -- 1.7.2.3