Compare commits

...
Author SHA1 Message Date
regnat a06cd93324 Split installTests
That way the CI will run the different ones on different runners
2022-02-25 08:33:44 +01:00
regnat 38edf8a0b4 Split the CI in several steps 2022-02-24 20:49:09 +01:00
regnat 55b46a5060 testS: poll more eagerly for the daemon start/stop
Polling every 1 second means that even the simplest test takes at least
2 seconds. We can reasonably poll 1/10 of that to make things much
  quicker (esp. given that most of the time 0.1s is enough for the
  daemon to be started or stopped)
2022-02-24 15:30:05 +01:00
regnat 8442d587ce Allow tracking the start and end time of tests 2022-02-24 15:30:05 +01:00
regnat 1e622145b2 Sort the tests by wall time
The tests are scheduled in the order they appear, so running the long
ones first slightly improves the scheduling.

On my machine, this decreases the time of `make install` from 40s to 36s
2022-02-24 15:30:05 +01:00
regnat 76c36742ee Show the elapsed time for each test
To have an idea of what’s taking time
2022-02-24 15:30:05 +01:00
6 changed files with 148 additions and 73 deletions

No files matched your search

+31 -4
View File
@@ -6,13 +6,15 @@ on:
jobs: jobs:
tests: build:
needs: [check_cachix] needs: [check_cachix]
strategy: strategy:
matrix: matrix:
os: [ubuntu-latest, macos-latest] os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
timeout-minutes: 60 timeout-minutes: 60
outputs:
flake-outputs-json: ${{ steps.list-outputs.outputs.json }}
steps: steps:
- uses: actions/checkout@v2.4.0 - uses: actions/checkout@v2.4.0
with: with:
@@ -25,7 +27,32 @@ jobs:
name: '${{ env.CACHIX_NAME }}' name: '${{ env.CACHIX_NAME }}'
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- run: nix --experimental-features 'nix-command flakes' flake check -L - run: nix --experimental-features 'nix-command flakes' build -L
- name: List all the tests to run
id: list-outputs
run: scripts/list-tests-flake-outptus-for-gha
test:
needs: [build, check_cachix]
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
outputName: ${{ fromJson(needs.build.outputs.flake-outputs-json) }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2.4.0
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v16
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- uses: cachix/cachix-action@v10
if: needs.check_cachix.outputs.secret == 'true'
with:
name: '${{ env.CACHIX_NAME }}'
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- run: nix --experimental-features 'nix-command flakes' build .#checks.$(nix eval --raw --impure --expr builtins.currentSystem).${{ matrix.outputName }} -L
check_cachix: check_cachix:
name: Cachix secret present for installer tests name: Cachix secret present for installer tests
@@ -40,7 +67,7 @@ jobs:
run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
installer: installer:
needs: [tests, check_cachix] needs: [test, check_cachix]
if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
@@ -76,7 +103,7 @@ jobs:
- run: nix-instantiate -E 'builtins.currentTime' --eval - run: nix-instantiate -E 'builtins.currentTime' --eval
docker_push_image: docker_push_image:
needs: [check_cachix, tests] needs: [check_cachix, build]
if: >- if: >-
github.event_name == 'push' && github.event_name == 'push' &&
github.ref_name == 'master' && github.ref_name == 'master' &&
+15 -13
View File
@@ -533,26 +533,28 @@
nixpkgs = nixpkgs-regression; nixpkgs = nixpkgs-regression;
}; };
installTests = forAllSystems (system: installTestsAgainstSelf = forAllSystems (system:
let pkgs = nixpkgsFor.${system}; in let pkgs = nixpkgsFor.${system}; in
pkgs.runCommand "install-tests" { testNixVersions pkgs pkgs.nix pkgs.pkgs.nix
againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; );
againstCurrentUnstable = installTestsAgainstCurrentUnstable = forAllSystems (system:
# FIXME: temporarily disable this on macOS because of #3605. let pkgs = nixpkgsFor.${system}; in
if system == "x86_64-linux" # FIXME: temporarily disable this on macOS because of #3605.
then testNixVersions pkgs pkgs.nix pkgs.nixUnstable if system == "x86_64-linux"
else null; then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
# Disabled because the latest stable version doesn't handle else pkgs.writeText "dummy" "dummy"
# `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work );
# againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; # Disabled because the latest stable version doesn't handle
} "touch $out"); # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
# againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
}; };
checks = forAllSystems (system: { checks = forAllSystems (system: {
binaryTarball = self.hydraJobs.binaryTarball.${system}; binaryTarball = self.hydraJobs.binaryTarball.${system};
perlBindings = self.hydraJobs.perlBindings.${system}; perlBindings = self.hydraJobs.perlBindings.${system};
installTests = self.hydraJobs.installTests.${system}; installTestsAgainstCurrentUnstable = self.hydraJobs.installTestsAgainstCurrentUnstable.${system};
installTestsAgainstSelf = self.hydraJobs.installTestsAgainstSelf.${system};
} // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) { } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
dockerImage = self.hydraJobs.dockerImage.${system}; dockerImage = self.hydraJobs.dockerImage.${system};
}); });
+12 -3
View File
@@ -7,6 +7,8 @@ green=""
yellow="" yellow=""
normal="" normal=""
TESTS_TIMER_LOG=${TESTS_TIMER_LOG:-/dev/null}
post_run_msg="ran test $1..." post_run_msg="ran test $1..."
if [ -t 1 ]; then if [ -t 1 ]; then
red="" red=""
@@ -15,14 +17,21 @@ if [ -t 1 ]; then
normal="" normal=""
fi fi
(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
start_time=$(date -u +%s)
echo "$(date -u +%s) $1 start" >> "$TESTS_TIMER_LOG"
log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
status=$? status=$?
echo "$(date -u +%s) $1 stop" >> "$TESTS_TIMER_LOG"
stop_time=$(date -u +%s)
elapsed_time=$(($stop_time-$start_time))
if [ $status -eq 0 ]; then if [ $status -eq 0 ]; then
echo "$post_run_msg [${green}PASS$normal]" echo "$post_run_msg [${green}PASS$normal] in ${elapsed_time}s"
elif [ $status -eq 99 ]; then elif [ $status -eq 99 ]; then
echo "$post_run_msg [${yellow}SKIP$normal]" echo "$post_run_msg [${yellow}SKIP$normal] after ${elapsed_time}s"
else else
echo "$post_run_msg [${red}FAIL$normal]" echo "$post_run_msg [${red}FAIL$normal] in ${elapsed_time}s"
echo "$log" | sed 's/^/ /' echo "$log" | sed 's/^/ /'
exit "$status" exit "$status"
fi fi
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
set -x
nix build --inputs-from . nixpkgs#jq -o jq
TEST_NAMES=$(nix flake show . --json | ./jq-bin/bin/jq -c ".checks[\"$(nix eval --raw --impure --expr builtins.currentSystem)\"] | keys")
echo "::set-output name=json::$TEST_NAMES"
+4 -4
View File
@@ -91,9 +91,9 @@ startDaemon() {
# ‘nix-daemon’ should have an option to fork into the background. # ‘nix-daemon’ should have an option to fork into the background.
rm -f $NIX_DAEMON_SOCKET_PATH rm -f $NIX_DAEMON_SOCKET_PATH
PATH=$DAEMON_PATH nix daemon & PATH=$DAEMON_PATH nix daemon &
for ((i = 0; i < 30; i++)); do for ((i = 0; i < 300; i++)); do
if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi
sleep 1 sleep 0.1
done done
pidDaemon=$! pidDaemon=$!
trap "killDaemon" EXIT trap "killDaemon" EXIT
@@ -102,9 +102,9 @@ startDaemon() {
killDaemon() { killDaemon() {
kill $pidDaemon kill $pidDaemon
for i in {0.10}; do for i in {0..100}; do
kill -0 $pidDaemon || break kill -0 $pidDaemon || break
sleep 1 sleep 0.1
done done
kill -9 $pidDaemon || true kill -9 $pidDaemon || true
wait $pidDaemon || true wait $pidDaemon || true
+75 -49
View File
@@ -1,72 +1,98 @@
nix_tests = \ nix_tests = \
hash.sh lang.sh add.sh simple.sh dependencies.sh \ flakes.sh \
config.sh \
gc.sh \
ca/gc.sh \ ca/gc.sh \
gc-concurrent.sh \ gc.sh \
gc-non-blocking.sh \ remote-store.sh \
lang.sh \
fetchMercurial.sh \
gc-auto.sh \ gc-auto.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ user-envs.sh \
gc-runtime.sh check-refs.sh filter-source.sh \
local-store.sh remote-store.sh export.sh export-graph.sh \
db-migration.sh \
timeout.sh secure-drv-outputs.sh nix-channel.sh \
multiple-outputs.sh import-derivation.sh ca/import-derivation.sh fetchurl.sh optimise-store.sh \
binary-cache.sh \ binary-cache.sh \
multiple-outputs.sh \
ca/build.sh \
nix-build.sh \
gc-concurrent.sh \
repair.sh \
fixed.sh \
export-graph.sh \
timeout.sh \
fetchGitRefs.sh \
gc-runtime.sh \
tarball.sh \
fetchGit.sh \
fetchurl.sh \
simple.sh \
referrers.sh \
optimise-store.sh \
substitute-with-invalid-ca.sh \ substitute-with-invalid-ca.sh \
binary-cache-build-remote.sh \ ca/concurrent-builds.sh \
nix-profile.sh repair.sh dump-db.sh case-hack.sh \ signing.sh \
check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ ca/build-with-garbage-path.sh \
placeholders.sh nix-shell.sh \ hash.sh \
linux-sandbox.sh \ gc-non-blocking.sh \
build-dry.sh \ check.sh \
ca/substitute.sh \
nix-shell.sh \
ca/signatures.sh \
ca/nix-shell.sh \
ca/nix-copy.sh \
check-refs.sh \
build-remote-input-addressed.sh \ build-remote-input-addressed.sh \
secure-drv-outputs.sh \
restricted.sh \
fetchGitSubmodules.sh \
flake-searching.sh \
ca/duplicate-realisation-in-closure.sh \
readfile-context.sh \
nix-channel.sh \
recursive.sh \
dependencies.sh \
check-reqs.sh \
build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-fixed.sh \
build-remote-content-addressed-floating.sh \ build-remote-content-addressed-floating.sh \
ssh-relay.sh \
nar-access.sh \ nar-access.sh \
pure-eval.sh \
ca/post-hook.sh \
repl.sh \
ca/repl.sh \
ca/recursive.sh \
binary-cache-build-remote.sh \
search.sh \
logging.sh \
export.sh \
config.sh \
add.sh \
local-store.sh \
filter-source.sh \
misc.sh \
dump-db.sh \
linux-sandbox.sh \
build-dry.sh \
structured-attrs.sh \ structured-attrs.sh \
fetchGit.sh \
fetchGitRefs.sh \
fetchGitSubmodules.sh \
fetchMercurial.sh \
signing.sh \
shell.sh \ shell.sh \
brotli.sh \ brotli.sh \
zstd.sh \ zstd.sh \
compression-levels.sh \ compression-levels.sh \
pure-eval.sh \
check.sh \
plugins.sh \
search.sh \
nix-copy-ssh.sh \ nix-copy-ssh.sh \
post-hook.sh \ post-hook.sh \
ca/post-hook.sh \
function-trace.sh \ function-trace.sh \
recursive.sh \
describe-stores.sh \
flakes.sh \
flake-local-settings.sh \ flake-local-settings.sh \
flake-searching.sh \
flake-bundler.sh \
build.sh \
repl.sh ca/repl.sh \
ca/build.sh \
ca/build-with-garbage-path.sh \
ca/duplicate-realisation-in-closure.sh \
ca/substitute.sh \
ca/signatures.sh \
ca/nix-shell.sh \
ca/nix-run.sh \
ca/recursive.sh \
ca/concurrent-builds.sh \
ca/nix-copy.sh \
eval-store.sh \ eval-store.sh \
readfile-context.sh \ why-depends.sh \
store-ping.sh \ import-derivation.sh \
ca/import-derivation.sh \
nix_path.sh \ nix_path.sh \
why-depends.sh case-hack.sh \
# parallel.sh placeholders.sh \
ssh-relay.sh \
plugins.sh \
build.sh \
ca/nix-run.sh \
db-migration.sh \
nix-profile.sh \
pass-as-file.sh \
describe-stores.sh \
store-ping.sh
ifeq ($(HAVE_LIBCPUID), 1) ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh nix_tests += compute-levels.sh