In preceding commits the ci/.sh scripts have lost most of their CI-specific assumptions. Let's go even further and explicitly support running ci/lib.sh outside of CI. This was possible before by faking up enough CI-specific variables, but as shown in the new "help" output being added here using the ci/lib.sh to provide "CI-like" has now become trivial. The ci/print-test-failures.sh scripts can now be used outside of CI as well, the only GitHub CI-specific part is now guarded by a check that we'll pass if outside of GitHub CI. There's also a special-case here to not clobber $MAKEFLAGS in the environment if we're outside of CI, in case the user has e.g. "jN" or other flags to "make" that they'd prefer configured already. Using "ci/lib.sh" as a stand-alone script is much more useful if it doesn't hardcode NPROC=10, let's provide a poor shellscript replacement for the online_cpus() we have in thread-utils.c to cover the most common OS's. It was suggested to use "2>&1" to invoke "command -v", but per my reading of [2] and my own testing that doesn't seem to be needed. Perhaps it's only needed for "which(1)"? 1. https://lore.kernel.org/git/214f8670-91d5-f4b6-efa1-76966c3ab1ee@xxxxxxxxxxxxxx/ 2. https://pubs.opengroup.org/onlinepubs/009604499/utilities/command.html Helped-by: Eric Sunshine <sunshine@xxxxxxxxxxxxxx> Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@xxxxxxxxx> --- ci/lib-ci-type.sh | 3 --- ci/lib-online_cpus.sh | 26 ++++++++++++++++++ ci/lib.sh | 57 ++++++++++++++++++++++++++++++++++++--- ci/print-test-failures.sh | 4 --- 4 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 ci/lib-online_cpus.sh diff --git a/ci/lib-ci-type.sh b/ci/lib-ci-type.sh index 6f01fd9e5d9..09acab7aaec 100644 --- a/ci/lib-ci-type.sh +++ b/ci/lib-ci-type.sh @@ -3,7 +3,4 @@ if test "$GITHUB_ACTIONS" = "true" then CI_TYPE=github-actions -else - echo "Could not identify CI type" >&2 - exit 1 fi diff --git a/ci/lib-online_cpus.sh b/ci/lib-online_cpus.sh new file mode 100644 index 00000000000..6ce7ec5692d --- /dev/null +++ b/ci/lib-online_cpus.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# TODO: Ideally we'd compile t/helper/test-online-cpus.c early, but +# that presents a chicken & egg problem. But if we move it to a +# stand-oline command... +online_cpus() { + NPROC= + + if command -v nproc >/dev/null + then + # GNU coreutils + NPROC=$(nproc) + elif command -v sysctl >/dev/null + then + # BSD & Mac OS X + NPROC=$(sysctl -n hw.ncpu) + elif test -n "$NUMBER_OF_PROCESSORS" + then + # Windows + NPROC="$NUMBER_OF_PROCESSORS" + else + NPROC=1 + fi + + echo $NPROC +} diff --git a/ci/lib.sh b/ci/lib.sh index 8f80be4c3df..362bd031df0 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -1,6 +1,30 @@ #!/bin/sh set -e +# Usage +CI_TYPE_HELP_COMMANDS=' + # run "make all test" like the "linux-leaks" job + (eval $(jobname=linux-leaks ci/lib.sh --all) && make test) + + # run "make all test" like the "linux-musl" job + (eval $(jobname=linux-musl ci/lib.sh --all) && make test) + + # run "make test" like the "linux-TEST-vars" job (uses various GIT_TEST_* modes) + make && (eval $(jobname=linux-TEST-vars ci/lib.sh --test) && make test) + + # run "make test" like the "linux-sha256" job + make && (eval $(jobname=linux-sha256 ci/lib.sh --test) && make test) +' + +CI_TYPE_HELP=" +running $0 outside of CI? You can use ci/lib.sh to set up your +environment like a given CI job. E.g.: +$CI_TYPE_HELP_COMMANDS + +note that some of these (e.g. the linux-musl one) may not work as +expected due to the CI job configuring a platform that may not match +yours." + # Helper libraries . ${0%/*}/lib-ci-type.sh @@ -9,6 +33,10 @@ mode=$1 if test -z "$mode" then echo "need a $0 mode, e.g. --build or --test" >&2 + if test -z "$CI_TYPE" + then + echo "$CI_TYPE_HELP" >&2 + fi exit 1 fi @@ -35,7 +63,7 @@ setenv () { do case "$1" in --build | --test) - if test "$1" != "$mode" + if test "$1" != "$mode" && test "$mode" != "--all" then skip=t fi @@ -65,6 +93,10 @@ setenv () { if test -n "$GITHUB_ENV" then echo "$key=$val" >>"$GITHUB_ENV" + elif test -z "$CI_TYPE" + then + echo "$key=\"$val\"" + echo "export $key" fi echo "SET: '$key=$val'" >&2 @@ -75,10 +107,25 @@ CC= # How many jobs to run in parallel? NPROC=10 +case "$CI_TYPE" in +'') + . ${0%/*}/lib-online_cpus.sh + NPROC=$(online_cpus) + + if test -n "$MAKEFLAGS" + then + COMMON_MAKEFLAGS="$MAKEFLAGS" + else + COMMON_MAKEFLAGS=--jobs=$NPROC + fi + ;; +*) + # For "--test" we carry the MAKEFLAGS over from earlier steps, except + # in stand-alone jobs which will use $COMMON_MAKEFLAGS. + COMMON_MAKEFLAGS=--jobs=$NPROC + ;; +esac -# For "--test" we carry the MAKEFLAGS over from earlier steps, except -# in stand-alone jobs which will use $COMMON_MAKEFLAGS. -COMMON_MAKEFLAGS=--jobs=$NPROC # Clear MAKEFLAGS that may come from the outside world. MAKEFLAGS=$COMMON_MAKEFLAGS @@ -98,6 +145,8 @@ github-actions) GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" setenv --test GIT_TEST_OPTS "$GIT_TEST_OPTS" ;; +'') + ;; *) echo "Unhandled CI type: $CI_TYPE" >&2 exit 1 diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 29f8c332eca..ec395c79ccd 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -36,10 +36,6 @@ do cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" ;; - *) - echo "Unhandled CI type: $CI_TYPE" >&2 - exit 1 - ;; esac fi done -- 2.36.0.rc2.898.gdf6bbac50ff
