Merge #5537: install nlohmann json headers (into 2.4-maintenance)

[Backport 2.4-maintenance] Release Notes 2.4: add `--indirect` no-op change

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v14
+    - uses: cachix/install-nix-action@v14.1
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v10
       if: needs.check_cachix.outputs.secret == 'true'
@@ -45,7 +45,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v14
+    - uses: cachix/install-nix-action@v14.1
     - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -63,7 +63,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2.3.4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v14
+    - uses: cachix/install-nix-action@v14.1
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"

.version

@@ -1 +1 @@
-2.4
+2.4.1

Makefile

@@ -10,6 +10,7 @@ makefiles = \
   src/libexpr/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
+  src/nlohmann/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \

doc/manual/local.mk

@@ -12,11 +12,13 @@ man-pages := $(foreach n, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
+# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
 	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
-	NIX_STATE_DIR=/dummy
+	NIX_STATE_DIR=/dummy \
+	NIX_CONFIG='cores = 0'
 
 nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 

doc/manual/src/SUMMARY.md.in

@@ -70,7 +70,7 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release 2.4 (2021-XX-XX)](release-notes/rl-2.4.md)
+  - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)
   - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md)

doc/manual/src/command-ref/nix-env.md

@@ -238,7 +238,16 @@ a number of possible ways:
 
 ## Examples
 
-To install a specific version of `gcc` from the active Nix expression:
+To install a package using a specific attribute path from the active Nix expression:
+
+```console
+$ nix-env -iA gcc40mips
+installing `gcc-4.0.2'
+$ nix-env -iA xorg.xorgserver
+installing `xorg-server-1.2.0'
+```
+
+To install a specific version of `gcc` using the derivation name:
 
 ```console
 $ nix-env --install gcc-3.3.2
@@ -246,6 +255,9 @@ installing `gcc-3.3.2'
 uninstalling `gcc-3.1'
 ```
 
+Using attribute path for selecting a package is preferred,
+as it is much faster and there will not be multiple matches.
+
 Note the previously installed version is removed, since
 `--preserve-installed` was not specified.
 
@@ -256,13 +268,6 @@ $ nix-env --install gcc
 installing `gcc-3.3.2'
 ```
 
-To install using a specific attribute:
-
-```console
-$ nix-env -i -A gcc40mips
-$ nix-env -i -A xorg.xorgserver
-```
-
 To install all derivations in the Nix expression `foo.nix`:
 
 ```console
@@ -374,22 +379,29 @@ For the other flags, see `--install`.
 ## Examples
 
 ```console
-$ nix-env --upgrade gcc
+$ nix-env --upgrade -A nixpkgs.gcc
 upgrading `gcc-3.3.1' to `gcc-3.4'
 ```
 
+When there are no updates available, nothing will happen:
+
 ```console
-$ nix-env -u gcc-3.3.2 --always (switch to a specific version)
+$ nix-env --upgrade -A nixpkgs.pan
+```
+
+Using `-A` is preferred when possible, as it is faster and unambiguous but
+it is also possible to upgrade to a specific version by matching the derivation name:
+
+```console
+$ nix-env -u gcc-3.3.2 --always
 upgrading `gcc-3.4' to `gcc-3.3.2'
 ```
 
-```console
-$ nix-env --upgrade pan
-(no upgrades available, so nothing happens)
-```
+To try to upgrade everything
+(matching packages based on the part of the derivation name without version):
 
 ```console
-$ nix-env -u (try to upgrade everything)
+$ nix-env -u
 upgrading `hello-2.1.2' to `hello-2.1.3'
 upgrading `mozilla-1.2' to `mozilla-1.4'
 ```

doc/manual/src/command-ref/opt-common.md

@@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options:
     }: ...
     ```
 
-    So if you call this Nix expression (e.g., when you do `nix-env -i
+    So if you call this Nix expression (e.g., when you do `nix-env -iA
     pkgname`), the function will be called automatically using the
     value [`builtins.currentSystem`](../expressions/builtins.md) for
     the `system` argument. You can override this using `--arg`, e.g.,
-    `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that
+    `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that
     since the argument is a Nix string literal, you have to escape the
     quotes.)
 

doc/manual/src/expressions/simple-building-testing.md

@@ -1,6 +1,6 @@
 # Building and Testing
 
-You can now try to build Hello. Of course, you could do `nix-env -i
+You can now try to build Hello. Of course, you could do `nix-env -f . -iA
 hello`, but you may not want to install a possibly broken package just
 yet. The best way to test the package is by using the command
 `nix-build`, which builds a Nix expression and creates a symlink named

doc/manual/src/introduction.md

@@ -76,7 +76,7 @@ there after an upgrade.  This means that you can _roll back_ to the
 old version:
 
 ```console
-$ nix-env --upgrade some-packages
+$ nix-env --upgrade -A nixpkgs.some-package
 $ nix-env --rollback
 ```
 
@@ -122,7 +122,7 @@ Nix expressions generally describe how to build a package from
 source, so an installation action like
 
 ```console
-$ nix-env --install firefox
+$ nix-env --install -A nixpkgs.firefox
 ```
 
 _could_ cause quite a bit of build activity, as not only Firefox but

doc/manual/src/package-management/basic-package-mgmt.md

@@ -24,7 +24,7 @@ collection; you could write your own Nix expressions based on Nixpkgs,
 or completely new ones.)
 
 You can manually download the latest version of Nixpkgs from
-<http://nixos.org/nixpkgs/download.html>. However, it’s much more
+<https://github.com/NixOS/nixpkgs>. However, it’s much more
 convenient to use the Nixpkgs [*channel*](channels.md), since it makes
 it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
 automatically added to your list of “subscribed” channels when you
@@ -47,41 +47,45 @@ $ nix-channel --update
 You can view the set of available packages in Nixpkgs:
 
 ```console
-$ nix-env -qa
-aterm-2.2
-bash-3.0
-binutils-2.15
-bison-1.875d
-blackdown-1.4.2
-bzip2-1.0.2
+$ nix-env -qaP
+nixpkgs.aterm                       aterm-2.2
+nixpkgs.bash                        bash-3.0
+nixpkgs.binutils                    binutils-2.15
+nixpkgs.bison                       bison-1.875d
+nixpkgs.blackdown                   blackdown-1.4.2
+nixpkgs.bzip2                       bzip2-1.0.2
 …
 ```
 
-The flag `-q` specifies a query operation, and `-a` means that you want
+The flag `-q` specifies a query operation, `-a` means that you want
 to show the “available” (i.e., installable) packages, as opposed to the
-installed packages. If you downloaded Nixpkgs yourself, or if you
-checked it out from GitHub, then you need to pass the path to your
-Nixpkgs tree using the `-f` flag:
+installed packages, and `-P` prints the attribute paths that can be used
+to unambiguously select a package for installation (listed in the first column).
+If you downloaded Nixpkgs yourself, or if you checked it out from GitHub,
+then you need to pass the path to your Nixpkgs tree using the `-f` flag:
 
 ```console
-$ nix-env -qaf /path/to/nixpkgs
+$ nix-env -qaPf /path/to/nixpkgs
+aterm                               aterm-2.2
+bash                                bash-3.0
+…
 ```
 
 where */path/to/nixpkgs* is where you’ve unpacked or checked out
 Nixpkgs.
 
-You can select specific packages by name:
+You can filter the packages by name:
 
 ```console
-$ nix-env -qa firefox
-firefox-34.0.5
-firefox-with-plugins-34.0.5
+$ nix-env -qaP firefox
+nixpkgs.firefox-esr                 firefox-91.3.0esr
+nixpkgs.firefox                     firefox-94.0.1
 ```
 
 and using regular expressions:
 
 ```console
-$ nix-env -qa 'firefox.*'
+$ nix-env -qaP 'firefox.*'
 ```
 
 It is also possible to see the *status* of available packages, i.e.,
@@ -89,11 +93,11 @@ whether they are installed into the user environment and/or present in
 the system:
 
 ```console
-$ nix-env -qas
+$ nix-env -qaPs
 …
--PS bash-3.0
---S binutils-2.15
-IPS bison-1.875d
+-PS  nixpkgs.bash                bash-3.0
+--S  nixpkgs.binutils            binutils-2.15
+IPS  nixpkgs.bison               bison-1.875d
 …
 ```
 
@@ -106,13 +110,13 @@ which is Nix’s mechanism for doing binary deployment. It just means that
 Nix knows that it can fetch a pre-built package from somewhere
 (typically a network server) instead of building it locally.
 
-You can install a package using `nix-env -i`. For instance,
+You can install a package using `nix-env -iA`. For instance,
 
 ```console
-$ nix-env -i subversion
+$ nix-env -iA nixpkgs.subversion
 ```
 
-will install the package called `subversion` (which is, of course, the
+will install the package called `subversion` from `nixpkgs` channel (which is, of course, the
 [Subversion version management system](http://subversion.tigris.org/)).
 
 > **Note**
@@ -122,7 +126,7 @@ will install the package called `subversion` (which is, of course, the
 > binary cache <https://cache.nixos.org>; it contains binaries for most
 > packages in Nixpkgs. Only if no binary is available in the binary
 > cache, Nix will build the package from source. So if `nix-env
-> -i subversion` results in Nix building stuff from source, then either
+> -iA nixpkgs.subversion` results in Nix building stuff from source, then either
 > the package is not built for your platform by the Nixpkgs build
 > servers, or your version of Nixpkgs is too old or too new. For
 > instance, if you have a very recent checkout of Nixpkgs, then the
@@ -133,7 +137,10 @@ will install the package called `subversion` (which is, of course, the
 > using a Git checkout of the Nixpkgs tree), you will get binaries for
 > most packages.
 
-Naturally, packages can also be uninstalled:
+Naturally, packages can also be uninstalled. Unlike when installing, you will
+need to use the derivation name (though the version part can be omitted),
+instead of the attribute path, as `nix-env` does not record which attribute
+was used for installing:
 
 ```console
 $ nix-env -e subversion
@@ -143,7 +150,7 @@ Upgrading to a new version is just as easy. If you have a new release of
 Nix Packages, you can do:
 
 ```console
-$ nix-env -u subversion
+$ nix-env -uA nixpkgs.subversion
 ```
 
 This will *only* upgrade Subversion if there is a “newer” version in the

doc/manual/src/package-management/binary-cache-substituter.md

@@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is
 not part of the Nix distribution, but you can install it from Nixpkgs:
 
 ```console
-$ nix-env -i nix-serve
+$ nix-env -iA nixpkgs.nix-serve
 ```
 
 You can then start the server, listening for HTTP connections on
@@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using
 `--option extra-binary-caches`, e.g.:
 
 ```console
-$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
+$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/
 ```
 
 The option `extra-binary-caches` tells Nix to use this binary cache in

doc/manual/src/package-management/profiles.md

@@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This
 would be what we would obtain if we had done
 
 ```console
-$ nix-env -i subversion
+$ nix-env -iA nixpkgs.subversion
 ```
 
 on a set of Nix expressions that contained Subversion 1.1.2.
@@ -54,7 +54,7 @@ environment is generated based on the current one. For instance,
 generation 43 was created from generation 42 when we did
 
 ```console
-$ nix-env -i subversion firefox
+$ nix-env -iA nixpkgs.subversion nixpkgs.firefox
 ```
 
 on a set of Nix expressions that contained Firefox and a new version of
@@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by
 (abbreviation `-p`):
 
 ```console
-$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
+$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion
 ```
 
 This will *not* change the `~/.nix-profile` symlink.

doc/manual/src/package-management/ssh-substituter.md

@@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are
 available on the server `avalon`:
 
 ```console
-$ nix-env -i firefox --substituters ssh://alice@avalon
+$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon
 ```
 
 This works similar to the binary cache substituter that Nix usually

doc/manual/src/quick-start.md

@@ -19,19 +19,19 @@ to subsequent chapters.
    channel:
 
    ```console
-   $ nix-env -qa
-   docbook-xml-4.3
-   docbook-xml-4.5
-   firefox-33.0.2
-   hello-2.9
-   libxslt-1.1.28
+   $ nix-env -qaP
+   nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
+   nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
+   nixpkgs.firefox                               firefox-33.0.2
+   nixpkgs.hello                                 hello-2.9
+   nixpkgs.libxslt                               libxslt-1.1.28
    …
    ```
 
 1. Install some packages from the channel:
 
    ```console
-   $ nix-env -i hello
+   $ nix-env -iA nixpkgs.hello
    ```
 
    This should download pre-built packages; it should not build them

doc/manual/src/release-notes/rl-2.4.md

@@ -1,4 +1,4 @@
-# Release 2.4 (2021-10-XX)
+# Release 2.4 (2021-11-01)
 
 This is the first release in more than two years and is the result of
 more than 2800 commits from 195 contributors since release 2.3.
@@ -276,18 +276,62 @@ more than 2800 commits from 195 contributors since release 2.3.
 
 * Plugins can now register `nix` subcommands.
 
+* The `--indirect` flag to `nix-store --add-root` has become a no-op.
+  `--add-root` will always generate indirect GC roots from now on.
+
 ## Incompatible changes
 
 * The `nix` command is now marked as an experimental feature. This
   means that you need to add
 
-  > experimental-features = nix-command
+  ```
+  experimental-features = nix-command
+  ```
 
   to your `nix.conf` if you want to use it, or pass
   `--extra-experimental-features nix-command` on the command line.
 
-* The old `nix run` has been renamed to `nix shell` (and there is a
-  new `nix run` that does something else, as described above).
+* The `nix` command no longer has a syntax for referring to packages
+  in a channel. This means that the following no longer works:
+
+  ```console
+  nix build nixpkgs.hello # Nix 2.3
+  ```
+
+  Instead, you can either use the `#` syntax to select a package from
+  a flake, e.g.
+
+  ```console
+  nix build nixpkgs#hello
+  ```
+
+  Or, if you want to use the `nixpkgs` channel in the `NIX_PATH`
+  environment variable:
+
+  ```console
+  nix build -f '<nixpkgs>' hello
+  ```
+
+* The old `nix run` has been renamed to `nix shell`, while there is a
+  new `nix run` that runs a default command. So instead of
+
+  ```console
+  nix run nixpkgs.hello -c hello # Nix 2.3
+  ```
+
+  you should use
+
+  ```console
+  nix shell nixpkgs#hello -c hello
+  ```
+
+  or just
+
+  ```console
+  nix run nixpkgs#hello
+  ```
+
+  if the command you want to run has the same name as the package.
 
 * It is now an error to modify the `plugin-files` setting via a
   command-line flag that appears after the first non-flag argument to

docker.nix

@@ -0,0 +1,253 @@
+{ pkgs ? import <nixpkgs> { }
+, lib ? pkgs.lib
+, name ? "nix"
+, tag ? "latest"
+, channelName ? "nixpkgs"
+, channelURL ? "https://nixos.org/channels/nixpkgs-unstable"
+}:
+let
+  defaultPkgs = with pkgs; [
+    nix
+    bashInteractive
+    coreutils-full
+    gnutar
+    gzip
+    gnugrep
+    which
+    curl
+    less
+    wget
+    man
+    cacert.out
+    findutils
+  ];
+
+  users = {
+
+    root = {
+      uid = 0;
+      shell = "/bin/bash";
+      home = "/root";
+      gid = 0;
+    };
+
+  } // lib.listToAttrs (
+    map
+      (
+        n: {
+          name = "nixbld${toString n}";
+          value = {
+            uid = 30000 + n;
+            gid = 30000;
+            groups = [ "nixbld" ];
+            description = "Nix build user ${toString n}";
+          };
+        }
+      )
+      (lib.lists.range 1 32)
+  );
+
+  groups = {
+    root.gid = 0;
+    nixbld.gid = 30000;
+  };
+
+  userToPasswd = (
+    k:
+    { uid
+    , gid ? 65534
+    , home ? "/var/empty"
+    , description ? ""
+    , shell ? "/bin/false"
+    , groups ? [ ]
+    }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}"
+  );
+  passwdContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs userToPasswd users))
+  );
+
+  userToShadow = k: { ... }: "${k}:!:1::::::";
+  shadowContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs userToShadow users))
+  );
+
+  # Map groups to members
+  # {
+  #   group = [ "user1" "user2" ];
+  # }
+  groupMemberMap = (
+    let
+      # Create a flat list of user/group mappings
+      mappings = (
+        builtins.foldl'
+          (
+            acc: user:
+              let
+                groups = users.${user}.groups or [ ];
+              in
+              acc ++ map
+                (group: {
+                  inherit user group;
+                })
+                groups
+          )
+          [ ]
+          (lib.attrNames users)
+      );
+    in
+    (
+      builtins.foldl'
+        (
+          acc: v: acc // {
+            ${v.group} = acc.${v.group} or [ ] ++ [ v.user ];
+          }
+        )
+        { }
+        mappings)
+  );
+
+  groupToGroup = k: { gid }:
+    let
+      members = groupMemberMap.${k} or [ ];
+    in
+    "${k}:x:${toString gid}:${lib.concatStringsSep "," members}";
+  groupContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs groupToGroup groups))
+  );
+
+  nixConf = {
+    sandbox = "false";
+    build-users-group = "nixbld";
+    trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
+  };
+  nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n";
+
+  baseSystem =
+    let
+      nixpkgs = pkgs.path;
+      channel = pkgs.runCommand "channel-nixos" { } ''
+        mkdir $out
+        ln -s ${nixpkgs} $out/nixpkgs
+        echo "[]" > $out/manifest.nix
+      '';
+      rootEnv = pkgs.buildPackages.buildEnv {
+        name = "root-profile-env";
+        paths = defaultPkgs;
+      };
+      manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
+        cat > $out <<EOF
+        [
+        ${lib.concatStringsSep "\n" (builtins.map (drv: let
+          outputs = drv.outputsToInstall or [ "out" ];
+        in ''
+          {
+            ${lib.concatStringsSep "\n" (builtins.map (output: ''
+              ${output} = { outPath = "${lib.getOutput output drv}"; };
+            '') outputs)}
+            outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
+            name = "${drv.name}";
+            outPath = "${drv}";
+            system = "${drv.system}";
+            type = "derivation";
+            meta = { };
+          }
+        '') defaultPkgs)}
+        ]
+        EOF
+      '';
+      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+        mkdir $out
+        cp -a ${rootEnv}/* $out/
+        ln -s ${manifest} $out/manifest.nix
+      '';
+    in
+    pkgs.runCommand "base-system"
+      {
+        inherit passwdContents groupContents shadowContents nixConfContents;
+        passAsFile = [
+          "passwdContents"
+          "groupContents"
+          "shadowContents"
+          "nixConfContents"
+        ];
+        allowSubstitutes = false;
+        preferLocalBuild = true;
+      } ''
+      env
+      set -x
+      mkdir -p $out/etc
+
+      cat $passwdContentsPath > $out/etc/passwd
+      echo "" >> $out/etc/passwd
+
+      cat $groupContentsPath > $out/etc/group
+      echo "" >> $out/etc/group
+
+      cat $shadowContentsPath > $out/etc/shadow
+      echo "" >> $out/etc/shadow
+
+      mkdir -p $out/usr
+      ln -s /nix/var/nix/profiles/share $out/usr/
+
+      mkdir -p $out/nix/var/nix/gcroots
+
+      mkdir $out/tmp
+
+      mkdir -p $out/etc/nix
+      cat $nixConfContentsPath > $out/etc/nix/nix.conf
+
+      mkdir -p $out/root
+      mkdir -p $out/nix/var/nix/profiles/per-user/root
+
+      ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
+      ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+      ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
+
+      ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
+      ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+
+      mkdir -p $out/root/.nix-defexpr
+      ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
+      echo "${channelURL} ${channelName}" > $out/root/.nix-channels
+
+      mkdir -p $out/bin $out/usr/bin
+      ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
+      ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
+    '';
+
+in
+pkgs.dockerTools.buildLayeredImageWithNixDb {
+
+  inherit name tag;
+
+  contents = [ baseSystem ];
+
+  extraCommands = ''
+    rm -rf nix-support
+    ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
+  '';
+
+  config = {
+    Cmd = [ "/root/.nix-profile/bin/bash" ];
+    Env = [
+      "USER=root"
+      "PATH=${lib.concatStringsSep ":" [
+        "/root/.nix-profile/bin"
+        "/nix/var/nix/profiles/default/bin"
+        "/nix/var/nix/profiles/default/sbin"
+      ]}"
+      "MANPATH=${lib.concatStringsSep ":" [
+        "/root/.nix-profile/share/man"
+        "/nix/var/nix/profiles/default/share/man"
+      ]}"
+      "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
+    ];
+  };
+
+}

flake.nix

@@ -14,7 +14,7 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
@@ -404,6 +404,13 @@
         installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
         installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
+        # docker image with Nix inside
+        dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system:
+          import ./docker.nix {
+            pkgs = nixpkgsFor.${system};
+            tag = version;
+          });
+
         # Line coverage analysis.
         coverage =
           with nixpkgsFor.x86_64-linux;

mk/libraries.mk

@@ -91,7 +91,7 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT)
 
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED)
+	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
     ifndef HOST_DARWIN
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
@@ -105,7 +105,7 @@ define build-library
     $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
 
     $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ -shared $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED)
+	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
     ifndef HOST_DARWIN
@@ -125,8 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
-	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
+	$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/programs.mk

@@ -32,7 +32,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	$$(trace-ld) $(CXX) -o $$@ $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS)
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
 
@@ -49,7 +49,7 @@ define build-program
       _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
 
       $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
-	$$(trace-ld) $(CXX) -o $$@ $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS)
+	$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     else
 

perl/lib/Nix/Config.pm.in

@@ -1,6 +1,7 @@
 package Nix::Config;
 
 use MIME::Base64;
+use Nix::Store;
 
 $version = "@PACKAGE_VERSION@";
 

scripts/install-darwin-multi-user.sh

@@ -78,7 +78,7 @@ poly_service_installed_check() {
 poly_service_uninstall_directions() {
     echo "$1. Remove macOS-specific components:"
     if should_create_volume && test_nix_volume_mountd_installed; then
-        darwin_volume_uninstall_directions
+        nix_volume_mountd_uninstall_directions
     fi
     if test_nix_daemon_installed; then
         nix_daemon_uninstall_directions

scripts/install-nix-from-closure.sh

@@ -38,7 +38,7 @@ fi
 
 # Determine if we could use the multi-user installer or not
 if [ "$(uname -s)" = "Linux" ]; then
-    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+    echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2
 fi
 
 case "$(uname -s)" in
@@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do
                 echo "              providing multi-user support and better isolation for local builds."
                 echo "              Both for security and reproducibility, this method is recommended if"
                 echo "              supported on your platform."
-                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo "              See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation"
                 echo ""
                 echo " --no-daemon: Simple, single-user installation that does not require root and is"
                 echo "              trivial to uninstall."
@@ -144,7 +144,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 
@@ -215,7 +215,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break
@@ -226,7 +226,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break

src/build-remote/build-remote.cc

@@ -130,11 +130,14 @@ static int main_build_remote(int argc, char * * argv)
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
-                    if (m.enabled && std::find(m.systemTypes.begin(),
-                            m.systemTypes.end(),
-                            neededSystem) != m.systemTypes.end() &&
+                    if (m.enabled
+                        && (neededSystem == "builtin"
+                            || std::find(m.systemTypes.begin(),
+                                m.systemTypes.end(),
+                                neededSystem) != m.systemTypes.end()) &&
                         m.allSupported(requiredFeatures) &&
-                        m.mandatoryMet(requiredFeatures)) {
+                        m.mandatoryMet(requiredFeatures))
+                    {
                         rightType = true;
                         AutoCloseFD free;
                         uint64_t load = 0;

src/libexpr/parser.y

@@ -752,7 +752,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, path };
         else {
             logWarning({
-                .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };
         }

src/libexpr/primops.cc

@@ -575,7 +575,7 @@ static Bindings::iterator getAttr(
             // Adding another trace for the function name to make it clear
             // which call received wrong arguments.
             e.addTrace(pos, hintfmt("while invoking '%s'", funcName));
-            throw;
+            throw e;
         }
     }
 

src/libexpr/primops/fetchTree.cc

@@ -74,7 +74,10 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau
 
 std::string fixURIForGit(std::string uri, EvalState & state)
 {
-    static std::regex scp_uri("([^/].*)@(.*):(.*)");
+    /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
+     * them by removing the `:` and assuming a scheme of `ssh://`
+     * */
+    static std::regex scp_uri("([^/]*)@(.*):(.*)");
     if (uri[0] != '/' && std::regex_match(uri, scp_uri))
         return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
     else

src/libmain/shared.cc

@@ -15,9 +15,14 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <signal.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
+#ifdef __linux__
+#include <features.h>
+#endif
+#ifdef __GLIBC__
+#include <gnu/lib-names.h>
+#include <nss.h>
+#include <dlfcn.h>
+#endif
 
 #include <openssl/crypto.h>
 
@@ -121,21 +126,30 @@ static void preloadNSS() {
        been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
        load its lookup libraries in the parent before any child gets a chance to. */
     std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN.
-           We need the resolution to be done locally, as nscd socket will not be accessible in the
-           sandbox. */
-        char * previous_env = getenv("LOCALDOMAIN");
-        setenv("LOCALDOMAIN", "invalid", 1);
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) {
-            if (res) freeaddrinfo(res);
-        }
-        if (previous_env) {
-             setenv("LOCALDOMAIN", previous_env, 1);
-        } else {
-             unsetenv("LOCALDOMAIN");
-        }
+#ifdef __GLIBC__
+        /* On linux, glibc will run every lookup through the nss layer.
+         * That means every lookup goes, by default, through nscd, which acts as a local
+         * cache.
+         * Because we run builds in a sandbox, we also remove access to nscd otherwise
+         * lookups would leak into the sandbox.
+         *
+         * But now we have a new problem, we need to make sure the nss_dns backend that
+         * does the dns lookups when nscd is not available is loaded or available.
+         *
+         * We can't make it available without leaking nix's environment, so instead we'll
+         * load the backend, and configure nss so it does not try to run dns lookups
+         * through nscd.
+         *
+         * This is technically only used for builtins:fetch* functions so we only care
+         * about dns.
+         *
+         * All other platforms are unaffected.
+         */
+        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
+            warn("unable to load nss_dns backend");
+        // FIXME: get hosts entry from nsswitch.conf.
+        __nss_configure_lookup("hosts", "files dns");
+#endif
     });
 }
 

src/libstore/build/derivation-goal.cc

@@ -616,7 +616,9 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+    bool buildLocally =
+        (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
+        && settings.maxBuildJobs.get() != 0;
 
     if (!buildLocally) {
         switch (tryBuildHook()) {
@@ -653,7 +655,7 @@ void DerivationGoal::tryLocalBuild() {
     throw Error(
         "unable to build with a primary store that isn't a local store; "
         "either pass a different '--store' or enable remote builds."
-        "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+        "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 }
 
 

src/libstore/build/local-derivation-goal.cc

@@ -948,7 +948,7 @@ void LocalDerivationGoal::startBuilder()
             FdSource source(builderOut.readSide.get());
             auto ex = readError(source);
             ex.addTrace({}, "while setting up the build environment");
-            throw;
+            throw ex;
         }
         debug("sandbox setup: " + msg);
         msgs.push_back(std::move(msg));

src/libstore/build/worker.cc

@@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals)
                 if (getMachines().empty())
                    throw Error("unable to start any build; either increase '--max-jobs' "
                             "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
                 else
                    throw Error("unable to start any build; remote machines may not have "
                             "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 
             }
             assert(!awake.empty());

src/libstore/daemon.cc

@@ -235,6 +235,7 @@ struct ClientSettings
                 }
                 else if (trusted
                     || name == settings.buildTimeout.name
+                    || name == settings.buildRepeat.name
                     || name == "connect-timeout"
                     || (name == "builders" && value == ""))
                     settings.set(name, value);
@@ -432,25 +433,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashType(hashAlgoRaw);
             }
 
-            StringSink saved;
-            TeeSource savedNARSource(from, saved);
-            RetrieveRegularNARSink savedRegular { saved };
-
-            if (method == FileIngestionMethod::Recursive) {
-                /* Get the entire NAR dump from the client and save it to
-                  a string so that we can pass it to
-                  addToStoreFromDump(). */
-                ParseSink sink; /* null sink; just parse the NAR */
-                parseDump(sink, savedNARSource);
-            } else
-                parseDump(savedRegular, from);
-
+            auto dumpSource = sinkToSource([&](Sink & saved) {
+                if (method == FileIngestionMethod::Recursive) {
+                    /* We parse the NAR dump through into `saved` unmodified,
+                       so why all this extra work? We still parse the NAR so
+                       that we aren't sending arbitrary data to `saved`
+                       unwittingly`, and we know when the NAR ends so we don't
+                       consume the rest of `from` and can't parse another
+                       command. (We don't trust `addToStoreFromDump` to not
+                       eagerly consume the entire stream it's given, past the
+                       length of the Nar. */
+                    TeeSource savedNARSource(from, saved);
+                    ParseSink sink; /* null sink; just parse the NAR */
+                    parseDump(sink, savedNARSource);
+                } else {
+                    /* Incrementally parse the NAR file, stripping the
+                       metadata, and streaming the sole file we expect into
+                       `saved`. */
+                    RetrieveRegularNARSink savedRegular { saved };
+                    parseDump(savedRegular, from);
+                    if (!savedRegular.regular) throw Error("regular file expected");
+                }
+            });
             logger->startWork();
-            if (!savedRegular.regular) throw Error("regular file expected");
-
-            // FIXME: try to stream directly from `from`.
-            StringSource dumpSource { *saved.s };
-            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
+            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);

src/libstore/filetransfer.cc

@@ -716,15 +716,32 @@ struct curlFileTransfer : public FileTransfer
     }
 };
 
+ref<curlFileTransfer> makeCurlFileTransfer()
+{
+    return make_ref<curlFileTransfer>();
+}
+
 ref<FileTransfer> getFileTransfer()
 {
-    static ref<FileTransfer> fileTransfer = makeFileTransfer();
+    static ref<curlFileTransfer> fileTransfer = makeCurlFileTransfer();
+
+    // this has to be done in its own scope to make sure that the lock is released
+    // before creating a new fileTransfer instance.
+    auto needsRecreation = [&]() -> bool {
+        auto state = fileTransfer->state_.lock();
+        return state->quit;
+    };
+
+    if (needsRecreation()) {
+        fileTransfer = makeCurlFileTransfer();
+    }
+
     return fileTransfer;
 }
 
 ref<FileTransfer> makeFileTransfer()
 {
-    return make_ref<curlFileTransfer>();
+    return makeCurlFileTransfer();
 }
 
 std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)

src/libstore/globals.cc

@@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms()
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") ||
+        pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
         if (std::string{SYSTEM} == "x86_64-darwin")
             extraPlatforms.insert("aarch64-darwin");
         else if (std::string{SYSTEM} == "aarch64-darwin")

src/libstore/local-store.cc

@@ -8,6 +8,7 @@
 #include "references.hh"
 #include "callback.hh"
 #include "topo-sort.hh"
+#include "finally.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -1327,13 +1328,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
         dump.resize(oldSize + want);
         auto got = 0;
+        Finally cleanup([&]() {
+            dump.resize(oldSize + got);
+        });
         try {
             got = source.read(dump.data() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
         }
-        dump.resize(oldSize + got);
     }
 
     std::unique_ptr<AutoDelete> delTempDir;

src/libstore/sandbox-defaults.sb

@@ -100,4 +100,5 @@
 
 ; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin.
 (allow file-read*
-       (subpath "/Library/Apple/usr/libexec/oah"))
+       (subpath "/Library/Apple/usr/libexec/oah")
+       (subpath "/System/Library/Apple/usr/libexec/oah"))

src/libstore/sqlite.cc

@@ -1,4 +1,5 @@
 #include "sqlite.hh"
+#include "globals.hh"
 #include "util.hh"
 
 #include <sqlite3.h>
@@ -27,8 +28,12 @@ namespace nix {
 
 SQLite::SQLite(const Path & path, bool create)
 {
+    // useSQLiteWAL also indicates what virtual file system we need.  Using
+    // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
+    // for Linux (WSL) where useSQLiteWAL should be false by default.
+    const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
     if (sqlite3_open_v2(path.c_str(), &db,
-            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
+            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK)
         throw Error("cannot open SQLite database '%s'", path);
 
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)

src/libstore/tests/local.mk

@@ -10,6 +10,6 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
 
 libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
 
-libstore-tests_LIBS = libstore
+libstore-tests_LIBS = libstore libutil
 
 libstore-tests_LDFLAGS := $(GTEST_LIBS)

src/libutil/ref.hh

@@ -17,7 +17,7 @@ private:
 
 public:
 
-    ref<T>(const ref<T> & r)
+    ref(const ref<T> & r)
         : p(r.p)
     { }
 

src/libutil/tests/tests.cc

@@ -4,6 +4,8 @@
 #include <limits.h>
 #include <gtest/gtest.h>
 
+#include <numeric>
+
 namespace nix {
 
 /* ----------- tests for util.hh ------------------------------------------------*/
@@ -282,6 +284,17 @@ namespace nix {
         ASSERT_EQ(decoded, s);
     }
 
+    TEST(base64Encode, encodeAndDecodeNonPrintable) {
+        char s[256];
+        std::iota(std::rbegin(s), std::rend(s), 0);
+
+        auto encoded = base64Encode(s);
+        auto decoded = base64Decode(encoded);
+
+        EXPECT_EQ(decoded.length(), 255);
+        ASSERT_EQ(decoded, s);
+    }
+
     /* ----------------------------------------------------------------------------
      * base64Decode
      * --------------------------------------------------------------------------*/
@@ -294,6 +307,10 @@ namespace nix {
         ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
     }
 
+    TEST(base64Decode, decodeThrowsOnInvalidChar) {
+        ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error);
+    }
+
     /* ----------------------------------------------------------------------------
      * toLower
      * --------------------------------------------------------------------------*/

src/libutil/util.cc

@@ -1436,8 +1436,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
 }
 
 
-static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-static std::array<char, 256> base64DecodeChars;
+constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 string base64Encode(std::string_view s)
 {
@@ -1462,12 +1461,15 @@ string base64Encode(std::string_view s)
 
 string base64Decode(std::string_view s)
 {
-    static std::once_flag flag;
-    std::call_once(flag, [](){
-        base64DecodeChars = { (char)-1 };
+    constexpr char npos = -1;
+    constexpr std::array<char, 256> base64DecodeChars = [&]() {
+        std::array<char, 256>  result{};
+        for (auto& c : result)
+            c = npos;
         for (int i = 0; i < 64; i++)
-            base64DecodeChars[(int) base64Chars[i]] = i;
-    });
+            result[base64Chars[i]] = i;
+        return result;
+    }();
 
     string res;
     unsigned int d = 0, bits = 0;
@@ -1477,7 +1479,7 @@ string base64Decode(std::string_view s)
         if (c == '\n') continue;
 
         char digit = base64DecodeChars[(unsigned char) c];
-        if (digit == -1)
+        if (digit == npos)
             throw Error("invalid character in Base64 string: '%c'", c);
 
         bits += 6;

src/nix-build/nix-build.cc

@@ -358,6 +358,7 @@ static void main_nix_build(int argc, char * * argv)
            is not set, then build bashInteractive from
            <nixpkgs>. */
         auto shell = getEnv("NIX_BUILD_SHELL");
+        std::optional<StorePath> shellDrv;
 
         if (!shell) {
 
@@ -374,8 +375,7 @@ static void main_nix_build(int argc, char * * argv)
                 auto bashDrv = store->parseStorePath(drv->queryDrvPath());
                 pathsToBuild.push_back({bashDrv});
                 pathsToCopy.insert(bashDrv);
-
-                shell = drv->queryOutPath() + "/bin/bash";
+                shellDrv = bashDrv;
 
             } catch (Error & e) {
                 logError(e.info());
@@ -401,6 +401,11 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
+        if (shellDrv) {
+            auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value());
+            shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
+        }
+
         if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
             auto resolvedDrv = drv.tryResolve(*store);
             assert(resolvedDrv && "Successfully resolved the derivation");

src/nix/repl.cc

@@ -200,13 +200,13 @@ namespace {
 void NixRepl::mainLoop(const std::vector<std::string> & files)
 {
     string error = ANSI_RED "error:" ANSI_NORMAL " ";
-    std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl;
+    notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n");
 
     for (auto & i : files)
         loadedFiles.push_back(i);
 
     reloadFiles();
-    if (!loadedFiles.empty()) std::cout << std::endl;
+    if (!loadedFiles.empty()) notice("");
 
     // Allow nix-repl specific settings in .inputrc
     rl_readline_name = "nix-repl";
@@ -396,6 +396,8 @@ bool NixRepl::processLine(string line)
 {
     if (line == "") return true;
 
+    _isInterrupted = false;
+
     string command, arg;
 
     if (line[0] == ':') {
@@ -479,9 +481,10 @@ bool NixRepl::processLine(string line)
     else if (command == ":t") {
         Value v;
         evalString(arg, v);
-        std::cout << showType(v) << std::endl;
+        logger->cout(showType(v));
+    }
 
-    } else if (command == ":u") {
+    else if (command == ":u") {
         Value v, f, result;
         evalString(arg, v);
         evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
@@ -498,17 +501,11 @@ bool NixRepl::processLine(string line)
         Path drvPathRaw = state->store->printStorePath(drvPath);
 
         if (command == ":b") {
-            /* We could do the build in this process using buildPaths(),
-               but doing it in a child makes it easier to recover from
-               problems / SIGINT. */
-            try {
-                runNix("nix", {"build", "--no-link", drvPathRaw});
-                auto drv = state->store->readDerivation(drvPath);
-                std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-                for (auto & i : drv.outputsAndOptPaths(*state->store))
-                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
-            } catch (ExecError &) {
-            }
+            state->store->buildPaths({DerivedPath::Built{drvPath}});
+            auto drv = state->store->readDerivation(drvPath);
+            logger->cout("\nThis derivation produced the following outputs:");
+            for (auto & i : drv.outputsAndOptPaths(*state->store))
+                logger->cout("  %s -> %s", i.first, state->store->printStorePath(*i.second.second));
         } else if (command == ":i") {
             runNix("nix-env", {"-i", drvPathRaw});
         } else {
@@ -541,9 +538,9 @@ bool NixRepl::processLine(string line)
                     + concatStringsSep(" ", args) + "\n\n";
             }
 
-            markdown += trim(stripIndentation(doc->doc));
+            markdown += stripIndentation(doc->doc);
 
-            std::cout << renderMarkdownToTerminal(markdown);
+            logger->cout(trim(renderMarkdownToTerminal(markdown)));
         } else
             throw Error("value does not have documentation");
     }
@@ -626,9 +623,9 @@ void NixRepl::reloadFiles()
 
     bool first = true;
     for (auto & i : old) {
-        if (!first) std::cout << std::endl;
+        if (!first) notice("");
         first = false;
-        std::cout << format("Loading '%1%'...") % i << std::endl;
+        notice("Loading '%1%'...", i);
         loadFile(i);
     }
 }
@@ -639,7 +636,7 @@ void NixRepl::addAttrsToScope(Value & attrs)
     state->forceAttrs(attrs);
     for (auto & i : *attrs.attrs)
         addVarToScope(i.name, *i.value);
-    std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl;
+    notice("Added %1% variables.", attrs.attrs->size());
 }
 
 

src/nlohmann/local.mk

@@ -0,0 +1,2 @@
+$(foreach i, $(wildcard src/nlohmann/*.hpp), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nlohmann, 0644)))

tests/ca-shell.nix

@@ -0,0 +1 @@
+{ ... }@args: import ./shell.nix (args // { contentAddressed = true; })

tests/common.sh.in

@@ -36,8 +36,9 @@ export PATH=@bindir@:$PATH
 if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then
   export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH
 fi
+DAEMON_PATH="$PATH"
 if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
-  export NIX_DAEMON_COMMAND="$NIX_DAEMON_PACKAGE/bin/nix-daemon"
+  DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH"
 fi
 coreutils=@coreutils@
 
@@ -89,7 +90,7 @@ startDaemon() {
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
     rm -f $NIX_DAEMON_SOCKET_PATH
-    ${NIX_DAEMON_COMMAND:-nix daemon} &
+    PATH=$DAEMON_PATH nix daemon &
     for ((i = 0; i < 30; i++)); do
         if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi
         sleep 1
@@ -126,7 +127,7 @@ isDaemonNewer () {
   [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
   local requiredVersion="$1"
   local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
-  return [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''2.4''") -ge 0 ]]
+  [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]]
 }
 
 requireDaemonNewerThan () {

tests/eval-store.sh

@@ -1,6 +1,8 @@
 source common.sh
 
-requireDaemonNewerThan "2.4pre20210727"
+# Using `--eval-store` with the daemon will eventually copy everything
+# to the build store, invalidating most of the tests here
+needLocalStore
 
 eval_store=$TEST_ROOT/eval-store
 

tests/nix-shell.sh

@@ -3,59 +3,53 @@ source common.sh
 clearStore
 
 if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
-    nix-shell () {
-        command nix-shell --arg contentAddressed true "$@"
-    }
-
-    nix_develop() {
-        nix develop --arg contentAddressed true "$@"
-    }
+    shellDotNix="$PWD/ca-shell.nix"
 else
-    nix_develop() {
-        nix develop "$@"
-    }
+    shellDotNix="$PWD/shell.nix"
 fi
 
+export NIX_PATH=nixpkgs="$shellDotNix"
+
 # Test nix-shell -A
 export IMPURE_VAR=foo
 export SELECTED_IMPURE_VAR=baz
-export NIX_BUILD_SHELL=$SHELL
-output=$(nix-shell --pure shell.nix -A shellDrv --run \
+
+output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"')
 
 [ "$output" = " - foo - bar - true" ]
 
 # Test --keep
-output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \
+output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"')
 
 [ "$output" = " - foo - bar - baz" ]
 
 # Test nix-shell on a .drv
-[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
+[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]]
 
-[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \
+[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]]
 
 # Test nix-shell on a .drv symlink
 
 # Legacy: absolute path and .drv extension required
-nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell.drv
+nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell.drv
 [[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # New behaviour: just needs to resolve to a derivation in the store
-nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell
+nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell
 [[ $(nix-shell --pure $TEST_ROOT/shell --run \
     'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]]
 
 # Test nix-shell -p
-output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"')
+output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"')
 [ "$output" = "foo bar" ]
 
 # Test nix-shell -p --arg x y
-output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"')
+output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"')
 [ "$output" = "baz" ]
 
 # Test nix-shell shebang mode
@@ -91,18 +85,18 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
 [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
 
 # Test 'nix develop'.
-nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]'
 
 # Ensure `nix develop -c` preserves stdin
-echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
+echo foo | nix develop -f "$shellDotNix" shellDrv -c cat | grep -q foo
 
 # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
-nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
+nix develop -f "$shellDotNix" shellDrv -c echo foo |& grep -q foo
 
 # Test 'nix print-dev-env'.
-[[ $(nix print-dev-env -f shell.nix shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]]
+[[ $(nix print-dev-env -f "$shellDotNix" shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]]
 
-source <(nix print-dev-env -f shell.nix shellDrv)
+source <(nix print-dev-env -f "$shellDotNix" shellDrv)
 [[ -n $stdenv ]]
 [[ ${arr1[2]} = "3 4" ]]
 [[ ${arr2[1]} = $'\n' ]]

tests/shell.nix

@@ -74,6 +74,10 @@ let pkgs = rec {
   '';
 
   bash = shell;
+  bashInteractive = runCommand "bash" {} ''
+    mkdir -p $out/bin
+    ln -s ${shell} $out/bin/bash
+  '';
 
   # ruby "interpreter" that outputs "$@"
   ruby = runCommand "ruby" {} ''

tests/structured-attrs.sh

@@ -2,7 +2,7 @@ source common.sh
 
 # 27ce722638 required some incompatible changes to the nix file, so skip this
 # tests for the older versions
-requireDaemonNewerThan "2.4pre20210622"
+requireDaemonNewerThan "2.4pre20210712"
 
 clearStore