Split installTests

That way the CI will run the different ones on different runners

.github/workflows/ci.yml

@@ -6,13 +6,15 @@ on:
 
 jobs:
 
-  tests:
+  build:
     needs: [check_cachix]
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
+    outputs:
+      flake-outputs-json: ${{ steps.list-outputs.outputs.json }}
     steps:
     - uses: actions/checkout@v2.4.0
       with:
@@ -25,7 +27,32 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    - run: nix --experimental-features 'nix-command flakes' flake check -L
+    - run: nix --experimental-features 'nix-command flakes' build -L
+    - name: List all the tests to run
+      id: list-outputs
+      run: scripts/list-tests-flake-outptus-for-gha
+
+  test:
+    needs: [build, check_cachix]
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        outputName: ${{ fromJson(needs.build.outputs.flake-outputs-json) }}
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v16
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - run: nix --experimental-features 'nix-command flakes' build .#checks.$(nix eval --raw --impure --expr builtins.currentSystem).${{ matrix.outputName }}  -L
+
 
   check_cachix:
     name: Cachix secret present for installer tests
@@ -40,7 +67,7 @@ jobs:
         run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
 
   installer:
-    needs: [tests, check_cachix]
+    needs: [test, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
     runs-on: ubuntu-latest
     outputs:
@@ -76,7 +103,7 @@ jobs:
     - run: nix-instantiate -E 'builtins.currentTime' --eval
 
   docker_push_image:
-    needs: [check_cachix, tests]
+    needs: [check_cachix, build]
     if: >-
       github.event_name == 'push' &&
       github.ref_name == 'master' &&

flake.nix

@@ -533,26 +533,28 @@
           nixpkgs = nixpkgs-regression;
         };
 
-        installTests = forAllSystems (system:
+        installTestsAgainstSelf = forAllSystems (system:
           let pkgs = nixpkgsFor.${system}; in
-          pkgs.runCommand "install-tests" {
-            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable =
-              # FIXME: temporarily disable this on macOS because of #3605.
-              if system == "x86_64-linux"
-              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
-              else null;
-            # Disabled because the latest stable version doesn't handle
-            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
-          } "touch $out");
+          testNixVersions pkgs pkgs.nix pkgs.pkgs.nix
+        );
+        installTestsAgainstCurrentUnstable = forAllSystems (system:
+          let pkgs = nixpkgsFor.${system}; in
+          # FIXME: temporarily disable this on macOS because of #3605.
+          if system == "x86_64-linux"
+          then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
+          else pkgs.writeText "dummy" "dummy"
+        );
+        # Disabled because the latest stable version doesn't handle
+        # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+        # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
 
       };
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
-        installTests = self.hydraJobs.installTests.${system};
+        installTestsAgainstCurrentUnstable = self.hydraJobs.installTestsAgainstCurrentUnstable.${system};
+        installTestsAgainstSelf = self.hydraJobs.installTestsAgainstSelf.${system};
       } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
       });

mk/run_test.sh

@@ -7,6 +7,8 @@ green=""
 yellow=""
 normal=""
 
+TESTS_TIMER_LOG=${TESTS_TIMER_LOG:-/dev/null}
+
 post_run_msg="ran test $1..."
 if [ -t 1 ]; then
     red=""
@@ -15,14 +17,21 @@ if [ -t 1 ]; then
     normal=""
 fi
 (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+
+start_time=$(date -u +%s)
+echo "$(date -u +%s) $1 start" >> "$TESTS_TIMER_LOG"
 log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
 status=$?
+echo "$(date -u +%s) $1 stop" >> "$TESTS_TIMER_LOG"
+stop_time=$(date -u +%s)
+elapsed_time=$(($stop_time-$start_time))
+
 if [ $status -eq 0 ]; then
-  echo "$post_run_msg [${green}PASS$normal]"
+  echo "$post_run_msg [${green}PASS$normal] in ${elapsed_time}s"
 elif [ $status -eq 99 ]; then
-  echo "$post_run_msg [${yellow}SKIP$normal]"
+  echo "$post_run_msg [${yellow}SKIP$normal] after ${elapsed_time}s"
 else
-  echo "$post_run_msg [${red}FAIL$normal]"
+  echo "$post_run_msg [${red}FAIL$normal] in ${elapsed_time}s"
   echo "$log" | sed 's/^/    /'
   exit "$status"
 fi

scripts/list-tests-flake-outptus-for-gha

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+set -x
+
+nix build --inputs-from . nixpkgs#jq -o jq
+
+TEST_NAMES=$(nix flake show . --json | ./jq-bin/bin/jq -c ".checks[\"$(nix eval --raw --impure --expr builtins.currentSystem)\"] | keys")
+
+
+echo "::set-output name=json::$TEST_NAMES"

tests/common.sh.in

@@ -91,9 +91,9 @@ startDaemon() {
     # ‘nix-daemon’ should have an option to fork into the background.
     rm -f $NIX_DAEMON_SOCKET_PATH
     PATH=$DAEMON_PATH nix daemon &
-    for ((i = 0; i < 30; i++)); do
+    for ((i = 0; i < 300; i++)); do
         if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi
-        sleep 1
+        sleep 0.1
     done
     pidDaemon=$!
     trap "killDaemon" EXIT
@@ -102,9 +102,9 @@ startDaemon() {
 
 killDaemon() {
     kill $pidDaemon
-    for i in {0.10}; do
+    for i in {0..100}; do
         kill -0 $pidDaemon || break
-        sleep 1
+        sleep 0.1
     done
     kill -9 $pidDaemon || true
     wait $pidDaemon || true

tests/local.mk

@@ -1,72 +1,98 @@
 nix_tests = \
-  hash.sh lang.sh add.sh simple.sh dependencies.sh \
-  config.sh \
-  gc.sh \
+  flakes.sh \
   ca/gc.sh \
-  gc-concurrent.sh \
-  gc-non-blocking.sh \
+  gc.sh \
+  remote-store.sh \
+  lang.sh \
+  fetchMercurial.sh \
   gc-auto.sh \
-  referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
-  gc-runtime.sh check-refs.sh filter-source.sh \
-  local-store.sh remote-store.sh export.sh export-graph.sh \
-  db-migration.sh \
-  timeout.sh secure-drv-outputs.sh nix-channel.sh \
-  multiple-outputs.sh import-derivation.sh ca/import-derivation.sh fetchurl.sh optimise-store.sh \
+  user-envs.sh \
   binary-cache.sh \
+  multiple-outputs.sh \
+  ca/build.sh \
+  nix-build.sh \
+  gc-concurrent.sh \
+  repair.sh \
+  fixed.sh \
+  export-graph.sh \
+  timeout.sh \
+  fetchGitRefs.sh \
+  gc-runtime.sh \
+  tarball.sh \
+  fetchGit.sh \
+  fetchurl.sh \
+  simple.sh \
+  referrers.sh \
+  optimise-store.sh \
   substitute-with-invalid-ca.sh \
-  binary-cache-build-remote.sh \
-  nix-profile.sh repair.sh dump-db.sh case-hack.sh \
-  check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
-  placeholders.sh nix-shell.sh \
-  linux-sandbox.sh \
-  build-dry.sh \
+  ca/concurrent-builds.sh \
+  signing.sh \
+  ca/build-with-garbage-path.sh \
+  hash.sh \
+  gc-non-blocking.sh \
+  check.sh \
+  ca/substitute.sh \
+  nix-shell.sh \
+  ca/signatures.sh \
+  ca/nix-shell.sh \
+  ca/nix-copy.sh \
+  check-refs.sh \
   build-remote-input-addressed.sh \
+  secure-drv-outputs.sh \
+  restricted.sh \
+  fetchGitSubmodules.sh \
+  flake-searching.sh \
+  ca/duplicate-realisation-in-closure.sh \
+  readfile-context.sh \
+  nix-channel.sh \
+  recursive.sh \
+  dependencies.sh \
+  check-reqs.sh \
   build-remote-content-addressed-fixed.sh \
   build-remote-content-addressed-floating.sh \
-  ssh-relay.sh \
   nar-access.sh \
+  pure-eval.sh \
+  ca/post-hook.sh \
+  repl.sh \
+  ca/repl.sh \
+  ca/recursive.sh \
+  binary-cache-build-remote.sh \
+  search.sh \
+  logging.sh \
+  export.sh \
+  config.sh \
+  add.sh \
+  local-store.sh \
+  filter-source.sh \
+  misc.sh \
+  dump-db.sh \
+  linux-sandbox.sh \
+  build-dry.sh \
   structured-attrs.sh \
-  fetchGit.sh \
-  fetchGitRefs.sh \
-  fetchGitSubmodules.sh \
-  fetchMercurial.sh \
-  signing.sh \
   shell.sh \
   brotli.sh \
   zstd.sh \
   compression-levels.sh \
-  pure-eval.sh \
-  check.sh \
-  plugins.sh \
-  search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
-  ca/post-hook.sh \
   function-trace.sh \
-  recursive.sh \
-  describe-stores.sh \
-  flakes.sh \
   flake-local-settings.sh \
-  flake-searching.sh \
-  flake-bundler.sh \
-  build.sh \
-  repl.sh ca/repl.sh \
-  ca/build.sh \
-  ca/build-with-garbage-path.sh \
-  ca/duplicate-realisation-in-closure.sh \
-  ca/substitute.sh \
-  ca/signatures.sh \
-  ca/nix-shell.sh \
-  ca/nix-run.sh \
-  ca/recursive.sh \
-  ca/concurrent-builds.sh \
-  ca/nix-copy.sh \
   eval-store.sh \
-  readfile-context.sh \
-  store-ping.sh \
+  why-depends.sh \
+  import-derivation.sh \
+  ca/import-derivation.sh \
   nix_path.sh \
-  why-depends.sh
-  # parallel.sh
+  case-hack.sh \
+  placeholders.sh \
+  ssh-relay.sh \
+  plugins.sh \
+  build.sh \
+  ca/nix-run.sh \
+  db-migration.sh \
+  nix-profile.sh \
+  pass-as-file.sh \
+  describe-stores.sh \
+  store-ping.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	nix_tests += compute-levels.sh