Merge branch 'sg/stress-test'
authorJunio C Hamano <gitster@pobox.com>
Fri, 18 Jan 2019 21:49:56 +0000 (13:49 -0800)
committerJunio C Hamano <gitster@pobox.com>
Fri, 18 Jan 2019 21:49:56 +0000 (13:49 -0800)
Flaky tests can now be repeatedly run under load with the
"--stress" option.

* sg/stress-test:
  test-lib: add the '--stress' option to run a test repeatedly under load
  test-lib-functions: introduce the 'test_set_port' helper function
  test-lib: set $TRASH_DIRECTORY earlier
  test-lib: consolidate naming of test-results paths
  test-lib: parse command line options earlier
  test-lib: parse options in a for loop to keep $@ intact
  test-lib: extract Bash version check for '-x' tracing
  test-lib: translate SIGTERM and SIGHUP to an exit

t/README
t/lib-git-daemon.sh
t/lib-git-p4.sh
t/lib-git-svn.sh
t/lib-httpd.sh
t/t0410-partial-clone.sh
t/t5512-ls-remote.sh
t/test-lib-functions.sh
t/test-lib.sh

index 28711cc..11ce767 100644 (file)
--- a/t/README
+++ b/t/README
@@ -186,6 +186,22 @@ appropriately before running "make".
        this feature by setting the GIT_TEST_CHAIN_LINT environment
        variable to "1" or "0", respectively.
 
+--stress::
+--stress=<N>::
+       Run the test script repeatedly in multiple parallel jobs until
+       one of them fails.  Useful for reproducing rare failures in
+       flaky tests.  The number of parallel jobs is, in order of
+       precedence: <N>, or the value of the GIT_TEST_STRESS_LOAD
+       environment variable, or twice the number of available
+       processors (as shown by the 'getconf' utility), or 8.
+       Implies `--verbose -x --immediate` to get the most information
+       about the failure.  Note that the verbose output of each test
+       job is saved to 't/test-results/$TEST_NAME.stress-<nr>.out',
+       and only the output of the failed test job is shown on the
+       terminal.  The names of the trash directories get a
+       '.stress-<nr>' suffix, and the trash directory of the failed
+       test job is renamed to end with a '.stress-failed' suffix.
+
 You can also set the GIT_TEST_INSTALLED environment variable to
 the bindir of an existing git installation to test that installation.
 You still need to have built this git sandbox, from which various
@@ -425,7 +441,8 @@ This test harness library does the following things:
  - Creates an empty test directory with an empty .git/objects database
    and chdir(2) into it.  This directory is 't/trash
    directory.$test_name_without_dotsh', with t/ subject to change by
-   the --root option documented above.
+   the --root option documented above, and a '.stress-<N>' suffix
+   appended by the --stress option.
 
  - Defines standard test helper functions for your scripts to
    use.  These functions are designed to make all scripts behave
index fd41229..79db3b7 100644 (file)
@@ -28,7 +28,7 @@ then
        test_skip_or_die $GIT_TEST_GIT_DAEMON "file system does not support FIFOs"
 fi
 
-LIB_GIT_DAEMON_PORT=${LIB_GIT_DAEMON_PORT-${this_test#t}}
+test_set_port LIB_GIT_DAEMON_PORT
 
 GIT_DAEMON_PID=
 GIT_DAEMON_DOCUMENT_ROOT_PATH="$PWD"/repo
index c275994..b3be3ba 100644 (file)
@@ -53,14 +53,7 @@ time_in_seconds () {
        (cd / && "$PYTHON_PATH" -c 'import time; print(int(time.time()))')
 }
 
-# Try to pick a unique port: guess a large number, then hope
-# no more than one of each test is running.
-#
-# This does not handle the case where somebody else is running the
-# same tests and has chosen the same ports.
-testid=${this_test#t}
-git_p4_test_start=9800
-P4DPORT=$((10669 + ($testid - $git_p4_test_start)))
+test_set_port P4DPORT
 
 P4PORT=localhost:$P4DPORT
 P4CLIENT=client
index a8130f9..f3b478c 100644 (file)
@@ -13,6 +13,7 @@ fi
 GIT_DIR=$PWD/.git
 GIT_SVN_DIR=$GIT_DIR/svn/refs/remotes/git-svn
 SVN_TREE=$GIT_SVN_DIR/svn-tree
+test_set_port SVNSERVE_PORT
 
 svn >/dev/null 2>&1
 if test $? -ne 1
@@ -119,7 +120,6 @@ require_svnserve () {
 }
 
 start_svnserve () {
-       SVNSERVE_PORT=${SVNSERVE_PORT-${this_test#t}}
        svnserve --listen-port $SVNSERVE_PORT \
                 --root "$rawsvnrepo" \
                 --listen-once \
index a8729f8..e465116 100644 (file)
@@ -82,7 +82,7 @@ case $(uname) in
 esac
 
 LIB_HTTPD_PATH=${LIB_HTTPD_PATH-"$DEFAULT_HTTPD_PATH"}
-LIB_HTTPD_PORT=${LIB_HTTPD_PORT-${this_test#t}}
+test_set_port LIB_HTTPD_PORT
 
 TEST_PATH="$TEST_DIRECTORY"/lib-httpd
 HTTPD_ROOT_PATH="$PWD"/httpd
index 169f7f1..bce0278 100755 (executable)
@@ -492,7 +492,6 @@ test_expect_success 'gc stops traversal when a missing but promised object is re
        ! grep "$TREE_HASH" out
 '
 
-LIB_HTTPD_PORT=12345  # default port, 410, cannot be used as non-root
 . "$TEST_DIRECTORY"/lib-httpd.sh
 start_httpd
 
index 32e722d..cd9e606 100755 (executable)
@@ -260,7 +260,7 @@ test_lazy_prereq GIT_DAEMON '
 # This test spawns a daemon, so run it only if the user would be OK with
 # testing with git-daemon.
 test_expect_success PIPE,JGIT,GIT_DAEMON 'indicate no refs in standards-compliant empty remote' '
-       JGIT_DAEMON_PORT=${JGIT_DAEMON_PORT-${this_test#t}} &&
+       test_set_port JGIT_DAEMON_PORT &&
        JGIT_DAEMON_PID= &&
        git init --bare empty.git &&
        >empty.git/git-daemon-export-ok &&
index 6b3bbf9..92cf8f8 100644 (file)
@@ -1263,3 +1263,42 @@ test_oid () {
        fi &&
        eval "printf '%s' \"\${$var}\""
 }
+
+# Choose a port number based on the test script's number and store it in
+# the given variable name, unless that variable already contains a number.
+test_set_port () {
+       local var=$1 port
+
+       if test $# -ne 1 || test -z "$var"
+       then
+               BUG "test_set_port requires a variable name"
+       fi
+
+       eval port=\$$var
+       case "$port" in
+       "")
+               # No port is set in the given env var, use the test
+               # number as port number instead.
+               # Remove not only the leading 't', but all leading zeros
+               # as well, so the arithmetic below won't (mis)interpret
+               # a test number like '0123' as an octal value.
+               port=${this_test#${this_test%%[1-9]*}}
+               if test "${port:-0}" -lt 1024
+               then
+                       # root-only port, use a larger one instead.
+                       port=$(($port + 10000))
+               fi
+               ;;
+       *[^0-9]*|0*)
+               error >&7 "invalid port number: $port"
+               ;;
+       *)
+               # The user has specified the port.
+               ;;
+       esac
+
+       # Make sure that parallel '--stress' test jobs get different
+       # ports.
+       port=$(($port + ${GIT_TEST_STRESS_JOB_NR:-0}))
+       eval $var=$port
+}
index c34831a..a1abb11 100644 (file)
@@ -71,19 +71,222 @@ then
        exit 1
 fi
 
+# Parse options while taking care to leave $@ intact, so we will still
+# have all the original command line options when executing the test
+# script again for '--tee' and '--verbose-log' below.
+store_arg_to=
+prev_opt=
+for opt
+do
+       if test -n "$store_arg_to"
+       then
+               eval $store_arg_to=\$opt
+               store_arg_to=
+               prev_opt=
+               continue
+       fi
+
+       case "$opt" in
+       -d|--d|--de|--deb|--debu|--debug)
+               debug=t ;;
+       -i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate)
+               immediate=t ;;
+       -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
+               GIT_TEST_LONG=t; export GIT_TEST_LONG ;;
+       -r)
+               store_arg_to=run_list
+               ;;
+       --run=*)
+               run_list=${opt#--*=} ;;
+       -h|--h|--he|--hel|--help)
+               help=t ;;
+       -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
+               verbose=t ;;
+       --verbose-only=*)
+               verbose_only=${opt#--*=}
+               ;;
+       -q|--q|--qu|--qui|--quie|--quiet)
+               # Ignore --quiet under a TAP::Harness. Saying how many tests
+               # passed without the ok/not ok details is always an error.
+               test -z "$HARNESS_ACTIVE" && quiet=t ;;
+       --with-dashes)
+               with_dashes=t ;;
+       --no-color)
+               color= ;;
+       --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
+               valgrind=memcheck
+               tee=t
+               ;;
+       --valgrind=*)
+               valgrind=${opt#--*=}
+               tee=t
+               ;;
+       --valgrind-only=*)
+               valgrind_only=${opt#--*=}
+               tee=t
+               ;;
+       --tee)
+               tee=t ;;
+       --root=*)
+               root=${opt#--*=} ;;
+       --chain-lint)
+               GIT_TEST_CHAIN_LINT=1 ;;
+       --no-chain-lint)
+               GIT_TEST_CHAIN_LINT=0 ;;
+       -x)
+               trace=t ;;
+       -V|--verbose-log)
+               verbose_log=t
+               tee=t
+               ;;
+       --stress)
+               stress=t ;;
+       --stress=*)
+               stress=${opt#--*=}
+               case "$stress" in
+               *[^0-9]*|0*|"")
+                       echo "error: --stress=<N> requires the number of jobs to run" >&2
+                       exit 1
+                       ;;
+               *)      # Good.
+                       ;;
+               esac
+               ;;
+       *)
+               echo "error: unknown test option '$opt'" >&2; exit 1 ;;
+       esac
+
+       prev_opt=$opt
+done
+if test -n "$store_arg_to"
+then
+       echo "error: $prev_opt requires an argument" >&2
+       exit 1
+fi
+
+if test -n "$valgrind_only"
+then
+       test -z "$valgrind" && valgrind=memcheck
+       test -z "$verbose" && verbose_only="$valgrind_only"
+elif test -n "$valgrind"
+then
+       test -z "$verbose_log" && verbose=t
+fi
+
+if test -n "$stress"
+then
+       verbose=t
+       trace=t
+       immediate=t
+fi
+
+TEST_STRESS_JOB_SFX="${GIT_TEST_STRESS_JOB_NR:+.stress-$GIT_TEST_STRESS_JOB_NR}"
+TEST_NAME="$(basename "$0" .sh)"
+TEST_RESULTS_DIR="$TEST_OUTPUT_DIRECTORY/test-results"
+TEST_RESULTS_BASE="$TEST_RESULTS_DIR/$TEST_NAME$TEST_STRESS_JOB_SFX"
+TRASH_DIRECTORY="trash directory.$TEST_NAME$TEST_STRESS_JOB_SFX"
+test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY"
+case "$TRASH_DIRECTORY" in
+/*) ;; # absolute path is good
+ *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;;
+esac
+
+# If --stress was passed, run this test repeatedly in several parallel loops.
+if test "$GIT_TEST_STRESS_STARTED" = "done"
+then
+       : # Don't stress test again.
+elif test -n "$stress"
+then
+       if test "$stress" != t
+       then
+               job_count=$stress
+       elif test -n "$GIT_TEST_STRESS_LOAD"
+       then
+               job_count="$GIT_TEST_STRESS_LOAD"
+       elif job_count=$(getconf _NPROCESSORS_ONLN 2>/dev/null) &&
+            test -n "$job_count"
+       then
+               job_count=$((2 * $job_count))
+       else
+               job_count=8
+       fi
+
+       mkdir -p "$TEST_RESULTS_DIR"
+       stressfail="$TEST_RESULTS_BASE.stress-failed"
+       rm -f "$stressfail"
+
+       stress_exit=0
+       trap '
+               kill $job_pids 2>/dev/null
+               wait
+               stress_exit=1
+       ' TERM INT HUP
+
+       job_pids=
+       job_nr=0
+       while test $job_nr -lt "$job_count"
+       do
+               (
+                       GIT_TEST_STRESS_STARTED=done
+                       GIT_TEST_STRESS_JOB_NR=$job_nr
+                       export GIT_TEST_STRESS_STARTED GIT_TEST_STRESS_JOB_NR
+
+                       trap '
+                               kill $test_pid 2>/dev/null
+                               wait
+                               exit 1
+                       ' TERM INT
+
+                       cnt=0
+                       while ! test -e "$stressfail"
+                       do
+                               $TEST_SHELL_PATH "$0" "$@" >"$TEST_RESULTS_BASE.stress-$job_nr.out" 2>&1 &
+                               test_pid=$!
+
+                               if wait $test_pid
+                               then
+                                       printf "OK   %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt
+                               else
+                                       echo $GIT_TEST_STRESS_JOB_NR >>"$stressfail"
+                                       printf "FAIL %2d.%d\n" $GIT_TEST_STRESS_JOB_NR $cnt
+                               fi
+                               cnt=$(($cnt + 1))
+                       done
+               ) &
+               job_pids="$job_pids $!"
+               job_nr=$(($job_nr + 1))
+       done
+
+       wait
+
+       if test -f "$stressfail"
+       then
+               echo "Log(s) of failed test run(s):"
+               for failed_job_nr in $(sort -n "$stressfail")
+               do
+                       echo "Contents of '$TEST_RESULTS_BASE.stress-$failed_job_nr.out':"
+                       cat "$TEST_RESULTS_BASE.stress-$failed_job_nr.out"
+               done
+               rm -rf "$TRASH_DIRECTORY.stress-failed"
+               # Move the last one.
+               mv "$TRASH_DIRECTORY.stress-$failed_job_nr" "$TRASH_DIRECTORY.stress-failed"
+       fi
+
+       exit $stress_exit
+fi
+
 # if --tee was passed, write the output not only to the terminal, but
 # additionally to the file test-results/$BASENAME.out, too.
-case "$GIT_TEST_TEE_STARTED, $* " in
-done,*)
-       # do not redirect again
-       ;;
-*' --tee '*|*' --va'*|*' -V '*|*' --verbose-log '*)
-       mkdir -p "$TEST_OUTPUT_DIRECTORY/test-results"
-       BASE="$TEST_OUTPUT_DIRECTORY/test-results/$(basename "$0" .sh)"
+if test "$GIT_TEST_TEE_STARTED" = "done"
+then
+       : # do not redirect again
+elif test -n "$tee"
+then
+       mkdir -p "$TEST_RESULTS_DIR"
 
        # Make this filename available to the sub-process in case it is using
        # --verbose-log.
-       GIT_TEST_TEE_OUTPUT_FILE=$BASE.out
+       GIT_TEST_TEE_OUTPUT_FILE=$TEST_RESULTS_BASE.out
        export GIT_TEST_TEE_OUTPUT_FILE
 
        # Truncate before calling "tee -a" to get rid of the results
@@ -91,11 +294,38 @@ done,*)
        >"$GIT_TEST_TEE_OUTPUT_FILE"
 
        (GIT_TEST_TEE_STARTED=done ${TEST_SHELL_PATH} "$0" "$@" 2>&1;
-        echo $? >"$BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE"
-       test "$(cat "$BASE.exit")" = 0
+        echo $? >"$TEST_RESULTS_BASE.exit") | tee -a "$GIT_TEST_TEE_OUTPUT_FILE"
+       test "$(cat "$TEST_RESULTS_BASE.exit")" = 0
        exit
-       ;;
-esac
+fi
+
+if test -n "$trace" && test -n "$test_untraceable"
+then
+       # '-x' tracing requested, but this test script can't be reliably
+       # traced, unless it is run with a Bash version supporting
+       # BASH_XTRACEFD (introduced in Bash v4.1).
+       #
+       # Perform this version check _after_ the test script was
+       # potentially re-executed with $TEST_SHELL_PATH for '--tee' or
+       # '--verbose-log', so the right shell is checked and the
+       # warning is issued only once.
+       if test -n "$BASH_VERSION" && eval '
+            test ${BASH_VERSINFO[0]} -gt 4 || {
+              test ${BASH_VERSINFO[0]} -eq 4 &&
+              test ${BASH_VERSINFO[1]} -ge 1
+            }
+          '
+       then
+               : Executed by a Bash version supporting BASH_XTRACEFD.  Good.
+       else
+               echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD"
+               trace=
+       fi
+fi
+if test -n "$trace" && test -z "$verbose_log"
+then
+       verbose=t
+fi
 
 # For repeatability, reset the environment to known value.
 # TERM is sanitized below, after saving color control sequences.
@@ -193,7 +423,7 @@ fi
 
 # Add libc MALLOC and MALLOC_PERTURB test
 # only if we are not executing the test with valgrind
-if expr " $GIT_TEST_OPTS " : ".* --valgrind " >/dev/null ||
+if test -n "$valgrind" ||
    test -n "$TEST_NO_MALLOC_CHECK"
 then
        setup_malloc_check () {
@@ -264,100 +494,6 @@ test "x$TERM" != "xdumb" && (
        ) &&
        color=t
 
-while test "$#" -ne 0
-do
-       case "$1" in
-       -d|--d|--de|--deb|--debu|--debug)
-               debug=t; shift ;;
-       -i|--i|--im|--imm|--imme|--immed|--immedi|--immedia|--immediat|--immediate)
-               immediate=t; shift ;;
-       -l|--l|--lo|--lon|--long|--long-|--long-t|--long-te|--long-tes|--long-test|--long-tests)
-               GIT_TEST_LONG=t; export GIT_TEST_LONG; shift ;;
-       -r)
-               shift; test "$#" -ne 0 || {
-                       echo 'error: -r requires an argument' >&2;
-                       exit 1;
-               }
-               run_list=$1; shift ;;
-       --run=*)
-               run_list=${1#--*=}; shift ;;
-       -h|--h|--he|--hel|--help)
-               help=t; shift ;;
-       -v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
-               verbose=t; shift ;;
-       --verbose-only=*)
-               verbose_only=${1#--*=}
-               shift ;;
-       -q|--q|--qu|--qui|--quie|--quiet)
-               # Ignore --quiet under a TAP::Harness. Saying how many tests
-               # passed without the ok/not ok details is always an error.
-               test -z "$HARNESS_ACTIVE" && quiet=t; shift ;;
-       --with-dashes)
-               with_dashes=t; shift ;;
-       --no-color)
-               color=; shift ;;
-       --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind)
-               valgrind=memcheck
-               shift ;;
-       --valgrind=*)
-               valgrind=${1#--*=}
-               shift ;;
-       --valgrind-only=*)
-               valgrind_only=${1#--*=}
-               shift ;;
-       --tee)
-               shift ;; # was handled already
-       --root=*)
-               root=${1#--*=}
-               shift ;;
-       --chain-lint)
-               GIT_TEST_CHAIN_LINT=1
-               shift ;;
-       --no-chain-lint)
-               GIT_TEST_CHAIN_LINT=0
-               shift ;;
-       -x)
-               # Some test scripts can't be reliably traced  with '-x',
-               # unless the test is run with a Bash version supporting
-               # BASH_XTRACEFD (introduced in Bash v4.1).  Check whether
-               # this test is marked as such, and ignore '-x' if it
-               # isn't executed with a suitable Bash version.
-               if test -z "$test_untraceable" || {
-                    test -n "$BASH_VERSION" && eval '
-                      test ${BASH_VERSINFO[0]} -gt 4 || {
-                        test ${BASH_VERSINFO[0]} -eq 4 &&
-                        test ${BASH_VERSINFO[1]} -ge 1
-                      }
-                    '
-                  }
-               then
-                       trace=t
-               else
-                       echo >&2 "warning: ignoring -x; '$0' is untraceable without BASH_XTRACEFD"
-               fi
-               shift ;;
-       -V|--verbose-log)
-               verbose_log=t
-               shift ;;
-       *)
-               echo "error: unknown test option '$1'" >&2; exit 1 ;;
-       esac
-done
-
-if test -n "$valgrind_only"
-then
-       test -z "$valgrind" && valgrind=memcheck
-       test -z "$verbose" && verbose_only="$valgrind_only"
-elif test -n "$valgrind"
-then
-       test -z "$verbose_log" && verbose=t
-fi
-
-if test -n "$trace" && test -z "$verbose_log"
-then
-       verbose=t
-fi
-
 if test -n "$color"
 then
        # Save the color control sequences now rather than run tput
@@ -476,7 +612,7 @@ die () {
 
 GIT_EXIT_OK=
 trap 'die' EXIT
-trap 'exit $?' INT
+trap 'exit $?' INT TERM HUP
 
 # The user-facing functions are loaded from a separate file so that
 # test_perf subshells can have them too
@@ -818,12 +954,9 @@ test_done () {
 
        if test -z "$HARNESS_ACTIVE"
        then
-               test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results"
-               mkdir -p "$test_results_dir"
-               base=${0##*/}
-               test_results_path="$test_results_dir/${base%.sh}.counts"
+               mkdir -p "$TEST_RESULTS_DIR"
 
-               cat >"$test_results_path" <<-EOF
+               cat >"$TEST_RESULTS_BASE.counts" <<-EOF
                total $test_count
                success $test_success
                fixed $test_fixed
@@ -1029,12 +1162,6 @@ then
 fi
 
 # Test repository
-TRASH_DIRECTORY="trash directory.$(basename "$0" .sh)"
-test -n "$root" && TRASH_DIRECTORY="$root/$TRASH_DIRECTORY"
-case "$TRASH_DIRECTORY" in
-/*) ;; # absolute path is good
- *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;;
-esac
 rm -fr "$TRASH_DIRECTORY" || {
        GIT_EXIT_OK=t
        echo >&5 "FATAL: Cannot prepare test area"