Remove dead code in doc’s local.mk

- Separate the generation of the manpages from their installation
- Make sure that `make` generates the manpages
- Make sure that `make install` installs them

Fix #5051

.github/workflows/test.yml

@@ -20,8 +20,7 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    #- run: nix flake check
-    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest

.gitignore

@@ -15,6 +15,7 @@ perl/Makefile.config
 /doc/manual/*.1
 /doc/manual/*.5
 /doc/manual/*.8
+/doc/manual/generated/*
 /doc/manual/nix.json
 /doc/manual/conf-file.json
 /doc/manual/builtins.json
@@ -76,7 +77,6 @@ perl/Makefile.config
 # /tests/
 /tests/test-tmp
 /tests/common.sh
-/tests/dummy
 /tests/result*
 /tests/restricted-innocent
 /tests/shell

Makefile

@@ -12,6 +12,7 @@ makefiles = \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \
+  misc/fish/local.mk \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
@@ -32,4 +33,4 @@ endif
 
 include mk/lib.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src

Makefile.config.in

@@ -1,3 +1,4 @@
+HOST_OS = @host_os@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
 BOOST_LDFLAGS = @BOOST_LDFLAGS@

boehmgc-coroutine-sp-fallback.diff

@@ -0,0 +1,42 @@
+diff --git a/pthread_stop_world.c b/pthread_stop_world.c
+index 1cee6a0b..46c3acd9 100644
+--- a/pthread_stop_world.c
++++ b/pthread_stop_world.c
+@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
+     struct GC_traced_stack_sect_s *traced_stack_sect;
+     pthread_t self = pthread_self();
+     word total_size = 0;
++    size_t stack_limit;
++    pthread_attr_t pattr;
+ 
+     if (!EXPECT(GC_thr_initialized, TRUE))
+       GC_thr_init();
+@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
+           hi = p->altstack + p->altstack_size;
+           /* FIXME: Need to scan the normal stack too, but how ? */
+           /* FIXME: Assume stack grows down */
++        } else {
++          if (pthread_getattr_np(p->id, &pattr)) {
++            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
++          }
++          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
++            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
++          }
++          // When a thread goes into a coroutine, we lose its original sp until
++          // control flow returns to the thread.
++          // While in the coroutine, the sp points outside the thread stack,
++          // so we can detect this and push the entire thread stack instead,
++          // as an approximation.
++          // We assume that the coroutine has similarly added its entire stack.
++          // This could be made accurate by cooperating with the application
++          // via new functions and/or callbacks.
++          #ifndef STACK_GROWS_UP
++            if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
++              lo = hi - stack_limit;
++            }
++          #else
++          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
++          #endif
+         }
+         GC_push_all_stack_sections(lo, hi, traced_stack_sect);
+ #       ifdef STACK_GROWS_UP

configure.ac

@@ -65,7 +65,7 @@ AC_SYS_LARGEFILE
 AC_STRUCT_DIRENT_D_TYPE
 if test "$sys_name" = sunos; then
     # Solaris requires -lsocket -lnsl for network functions
-    LIBS="-lsocket -lnsl $LIBS"
+    LDFLAGS="-lsocket -lnsl $LDFLAGS"
 fi
 
 
@@ -150,7 +150,7 @@ int main() {
 }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
 AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
 if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
-    LIBS="-latomic $LIBS"
+    LDFLAGS="-latomic $LDFLAGS"
 fi
 
 PKG_PROG_PKG_CONFIG
@@ -231,7 +231,7 @@ AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
-  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], 
+  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1],
   [AC_DEFINE([ENABLE_S3], [0], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=])
 AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)

doc/manual/generate-builtins.nix

@@ -6,9 +6,11 @@ builtins:
 concatStrings (map
   (name:
     let builtin = builtins.${name}; in
-    "  - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args)
-    + "  \n\n"
-    + concatStrings (map (s: "    ${s}\n") (splitLines builtin.doc)) + "\n\n"
+    "<dt><code>${name} "
+    + concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
+    + "</code></dt>"
+    + "<dd>\n\n"
+    + builtin.doc
+    + "\n\n</dd>"
   )
   (attrNames builtins))
-

doc/manual/local.mk

@@ -1,7 +1,5 @@
 ifeq ($(doc_generate),yes)
 
-MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
-
 # Generate man pages.
 man-pages := $(foreach n, \
   nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
@@ -64,6 +62,7 @@ $(d)/conf-file.json: $(bindir)/nix
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	@cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/builtins.json: $(bindir)/nix
@@ -74,14 +73,26 @@ $(d)/builtins.json: $(bindir)/nix
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
-install: $(d)/src/command-ref/new-cli
+install: $(mandir)/man1/nix3-manpages
+man: doc/manual/generated/man1/nix3-manpages
+all: doc/manual/generated/man1/nix3-manpages
+
+$(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
+	@mkdir -p $$(dirname $@)
+	$(trace-install) install -m 0644 $$(dirname $<)/* $$(dirname $@)
+
+doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
+	@mkdir -p $$(dirname $@)
 	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
 	  name=$$(basename $$i .md); \
+	  tmpFile=$$(mktemp); \
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
-	  printf "Title: %s\n\n" "$$name" > $$i.tmp; \
-	  cat $$i >> $$i.tmp; \
-	  lowdown -sT man -M section=1 $$i.tmp -o $(mandir)/man1/$$name.1; \
+	  printf "Title: %s\n\n" "$$name" > $$tmpFile; \
+	  cat $$i >> $$tmpFile; \
+	  lowdown -sT man -M section=1 $$tmpFile -o $$(dirname $@)/$$name.1; \
+	  rm $$tmpFile; \
 	done
+	touch $@
 
 $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
 	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual

doc/manual/src/command-ref/env-common.md

@@ -10,35 +10,39 @@ Most Nix commands interpret the following environment variables:
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
-    
+
         /home/eelco/Dev:/etc/nixos
-    
+
     will cause Nix to look for paths relative to `/home/eelco/Dev` and
     `/etc/nixos`, in this order. It is also possible to match paths
     against a prefix. For example, the value
-    
+
         nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
-    
+
     will cause Nix to search for `<nixpkgs/path>` in
     `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-    
+
     If a path in the Nix search path starts with `http://` or
     `https://`, it is interpreted as the URL of a tarball that will be
     downloaded and unpacked to a temporary location. The tarball must
     consist of a single top-level directory. For example, setting
     `NIX_PATH` to
-    
-        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
-    
-    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
-    channel.
-    
-    A following shorthand can be used to refer to the official channels:
-    
-        nixpkgs=channel:nixos-15.09
-    
-    The search path can be extended using the `-I` option, which takes
-    precedence over `NIX_PATH`.
+
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
+
+    tells Nix to download and use the current contents of the
+    `master` branch in the `nixpkgs` repository.
+
+    The URLs of the tarballs from the official nixos.org channels (see
+    [the manual for `nix-channel`](nix-channel.md)) can be abbreviated
+    as `channel:<channel-name>`.  For instance, the following two
+    values of `NIX_PATH` are equivalent:
+
+        nixpkgs=channel:nixos-21.05
+        nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
+
+    The Nix search path can also be extended using the `-I` option to
+    many Nix commands, which takes precedence over `NIX_PATH`.
 
   - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
@@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables:
     builds are deployed to machines where `/nix/store` resolves
     differently. If you are sure that you’re not going to do that, you
     can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
-    
+
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
@@ -59,7 +63,7 @@ Most Nix commands interpret the following environment variables:
     $ mkdir /nix
     $ mount -o bind /mnt/otherdisk/nix /nix
     ```
-    
+
     Consult the mount 8 manual page for details.
 
   - `NIX_STORE_DIR`\

doc/manual/src/expressions/builtins-prefix.md

@@ -9,7 +9,8 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `derivation` *attrs*; `builtins.derivation` *attrs*\
-
-    `derivation` is described in [its own section](derivations.md).
-
+<dl>
+  <dt><code>derivation <var>attrs</var></code>;
+      <code>builtins.derivation <var>attrs</var></code></dt>
+  <dd><p><var>derivation</var> in described in
+         <a href="derivations.md">its own section</a>.</p></dd>

doc/manual/src/expressions/builtins-suffix.md

@@ -0,0 +1 @@
+</dl>

flake.lock

@@ -19,11 +19,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1622593737,
-        "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=",
+        "lastModified": 1624862269,
+        "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e",
+        "rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
         "type": "github"
       },
       "original": {

flake.nix

@@ -20,6 +20,8 @@
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
       systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ];
 
+      crossSystems = [ "armv6l-linux" "armv7l-linux" ];
+
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
 
       # Memoize nixpkgs for different platforms for efficiency.
@@ -79,7 +81,7 @@
             buildPackages.mercurial
             buildPackages.jq
           ]
-          ++ lib.optionals stdenv.isLinux [(pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
 
         buildDeps =
           [ curl
@@ -87,13 +89,12 @@
             openssl sqlite
             libarchive
             boost
-            nlohmann_json
             lowdown
             gmock
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-          ++ lib.optional stdenv.isx86_64 libcpuid;
+          ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
 
         awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
           (aws-sdk-cpp.override {
@@ -102,7 +103,13 @@
           });
 
         propagatedDeps =
-          [ (boehmgc.override { enableLargeConfig = true; })
+          [ ((boehmgc.override {
+              enableLargeConfig = true;
+            }).overrideAttrs(o: {
+              patches = (o.patches or []) ++ [
+                ./boehmgc-coroutine-sp-fallback.diff
+              ];
+            }))
           ];
 
         perlDeps =
@@ -133,10 +140,11 @@
 
             substitute ${./scripts/install.in} $out/install \
               ${pkgs.lib.concatMapStrings
-                (system:
-                  '' \
-                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                  --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                (system: let
+                    tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system};
+                  in '' \
+                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
+                  --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
                   ''
                 )
                 systems
@@ -174,6 +182,77 @@
 
       };
 
+      binaryTarball = buildPackages: nix: pkgs: let
+        inherit (pkgs) cacert;
+        installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
+      in
+
+      buildPackages.runCommand "nix-binary-tarball-${version}"
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+          meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
+        }
+        ''
+          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+          substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+
+          if type -p shellcheck; then
+            # SC1090: Don't worry about not being able to find
+            #         $nix/etc/profile.d/nix.sh
+            shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/create-darwin-volume.sh
+            shellcheck $TMPDIR/install-darwin-multi-user.sh
+            shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+            # SC1091: Don't panic about not being able to source
+            #         /etc/profile
+            # SC2002: Ignore "useless cat" "error", when loading
+            #         .reginfo, as the cat is a much cleaner
+            #         implementation, even though it is "useless"
+            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+            #         root's home directory
+            shellcheck --external-sources \
+              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+          fi
+
+          chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/create-darwin-volume.sh
+          chmod +x $TMPDIR/install-darwin-multi-user.sh
+          chmod +x $TMPDIR/install-systemd-multi-user.sh
+          chmod +x $TMPDIR/install-multi-user
+          dir=nix-${version}-${pkgs.system}
+          fn=$out/$dir.tar.xz
+          mkdir -p $out/nix-support
+          echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+          tar cvfJ $fn \
+            --owner=0 --group=0 --mode=u+rw,uga+r \
+            --absolute-names \
+            --hard-dereference \
+            --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
+            --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+            --transform "s,$NIX_STORE,$dir/store,S" \
+            $TMPDIR/install \
+            $TMPDIR/create-darwin-volume.sh \
+            $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install-systemd-multi-user.sh \
+            $TMPDIR/install-multi-user \
+            $TMPDIR/reginfo \
+            $(cat ${installerClosureInfo}/store-paths)
+        '';
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
@@ -254,9 +333,9 @@
                 xz
                 pkgs.perl
                 boost
-                nlohmann_json
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -284,7 +363,7 @@
 
           outputs = [ "out" "bin" "dev" ];
 
-          nativeBuildInputs = [ which ];
+          nativeBuildInputs = [ buildPackages.which ];
 
           configurePhase = ''
               ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
@@ -303,92 +382,33 @@
 
         buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
+        buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem:
+          nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
+
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
 
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
-        binaryTarball = nixpkgs.lib.genAttrs systems (system:
+        binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system});
 
-          with nixpkgsFor.${system};
-
-          let
-            installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
-          in
-
-          runCommand "nix-binary-tarball-${version}"
-            { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-              meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
-            }
-            ''
-              cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-              cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-              substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              if type -p shellcheck; then
-                # SC1090: Don't worry about not being able to find
-                #         $nix/etc/profile.d/nix.sh
-                shellcheck --exclude SC1090 $TMPDIR/install
-                shellcheck $TMPDIR/create-darwin-volume.sh
-                shellcheck $TMPDIR/install-darwin-multi-user.sh
-                shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-                # SC1091: Don't panic about not being able to source
-                #         /etc/profile
-                # SC2002: Ignore "useless cat" "error", when loading
-                #         .reginfo, as the cat is a much cleaner
-                #         implementation, even though it is "useless"
-                # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-                #         root's home directory
-                shellcheck --external-sources \
-                  --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-              fi
-
-              chmod +x $TMPDIR/install
-              chmod +x $TMPDIR/create-darwin-volume.sh
-              chmod +x $TMPDIR/install-darwin-multi-user.sh
-              chmod +x $TMPDIR/install-systemd-multi-user.sh
-              chmod +x $TMPDIR/install-multi-user
-              dir=nix-${version}-${system}
-              fn=$out/$dir.tar.xz
-              mkdir -p $out/nix-support
-              echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-              tar cvfJ $fn \
-                --owner=0 --group=0 --mode=u+rw,uga+r \
-                --absolute-names \
-                --hard-dereference \
-                --transform "s,$TMPDIR/install,$dir/install," \
-                --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
-                --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-                --transform "s,$NIX_STORE,$dir/store,S" \
-                $TMPDIR/install \
-                $TMPDIR/create-darwin-volume.sh \
-                $TMPDIR/install-darwin-multi-user.sh \
-                $TMPDIR/install-systemd-multi-user.sh \
-                $TMPDIR/install-multi-user \
-                $TMPDIR/reginfo \
-                $(cat ${installerClosureInfo}/store-paths)
-            '');
+        binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: {
+          name = crossSystem;
+          value = let
+            nixpkgsCross = import nixpkgs {
+              inherit system crossSystem;
+              overlays = [ self.overlay ];
+            };
+          in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross;
+        }) crossSystems));
 
         # The first half of the installation script. This is uploaded
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
+        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
         # Line coverage analysis.
         coverage =
@@ -482,11 +502,14 @@
             # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
             # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
           } "touch $out";
-      });
+      } // (if system == "x86_64-linux" then (builtins.listToAttrs (map (crossSystem: {
+        name = "binaryTarball-${crossSystem}";
+        value = self.hydraJobs.binaryTarballCross.${system}.${crossSystem};
+      }) crossSystems)) else {}));
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;
-      } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
+      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;
         in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {
@@ -524,8 +547,49 @@
           stripAllList = ["bin"];
 
           strictDeps = true;
+
+          hardeningDisable = [ "pie" ];
         };
-      });
+      } // builtins.listToAttrs (map (crossSystem: {
+        name = "nix-${crossSystem}";
+        value = let
+          nixpkgsCross = import nixpkgs {
+            inherit system crossSystem;
+            overlays = [ self.overlay ];
+          };
+        in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation {
+          name = "nix-${version}";
+
+          src = self;
+
+          VERSION_SUFFIX = versionSuffix;
+
+          outputs = [ "out" "dev" "doc" ];
+
+          nativeBuildInputs = nativeBuildDeps;
+          buildInputs = buildDeps ++ propagatedDeps;
+
+          configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ];
+
+          enableParallelBuilding = true;
+
+          makeFlags = "profiledir=$(out)/etc/profile.d";
+
+          doCheck = true;
+
+          installFlags = "sysconfdir=$(out)/etc";
+
+          postInstall = ''
+            mkdir -p $doc/nix-support
+            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+            mkdir -p $out/nix-support
+            echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
+          '';
+
+          doInstallCheck = true;
+          installCheckFlags = "sysconfdir=$(out)/etc";
+        };
+      }) crossSystems)));
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 

maintainers/upload-release.pl

@@ -110,6 +110,9 @@ downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("binaryTarball.aarch64-darwin", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
 downloadFile("installerScript", "1");
 
 for my $fn (glob "$tmpDir/*") {
@@ -153,6 +156,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
            "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
            "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
            "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
            "}\n");
 
 system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;

misc/fish/completion.fish

@@ -0,0 +1,37 @@
+function _nix_complete
+  # Get the current command up to a cursor.
+  # - Behaves correctly even with pipes and nested in commands like env.
+  # - TODO: Returns the command verbatim (does not interpolate variables).
+  #   That might not be optimal for arguments like -f.
+  set -l nix_args (commandline --current-process --tokenize --cut-at-cursor)
+  # --cut-at-cursor with --tokenize removes the current token so we need to add it separately.
+  # https://github.com/fish-shell/fish-shell/issues/7375
+  # Can be an empty string.
+  set -l current_token (commandline --current-token --cut-at-cursor)
+
+  # Nix wants the index of the argv item to complete but the $nix_args variable
+  # also contains the program name (argv[0]) so we would need to subtract 1.
+  # But the variable also misses the current token so it cancels out.
+  set -l nix_arg_to_complete (count $nix_args)
+
+  env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token
+end
+
+function _nix_accepts_files
+  set -l response (_nix_complete)
+  # First line is either filenames or no-filenames.
+  test $response[1] = 'filenames'
+end
+
+function _nix
+  set -l response (_nix_complete)
+  # Skip the first line since it handled by _nix_accepts_files.
+  # Tail lines each contain a command followed by a tab character and, optionally, a description.
+  # This is also the format fish expects.
+  string collect -- $response[2..-1]
+end
+
+# Disable file path completion if paths do not belong in the current context.
+complete --command nix --condition 'not _nix_accepts_files' --no-files
+
+complete --command nix --arguments '(_nix)'

misc/fish/local.mk

@@ -0,0 +1 @@
+$(eval $(call install-file-as, $(d)/completion.fish, $(datarootdir)/fish/vendor_completions.d/nix.fish, 0644))

misc/launchd/local.mk

@@ -1,4 +1,4 @@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 
   $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons))
 

misc/systemd/local.mk

@@ -1,4 +1,4 @@
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
 

misc/upstart/local.mk

@@ -1,4 +1,4 @@
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644)))
 

misc/zsh/completion.zsh

@@ -1,3 +1,5 @@
+#compdef nix
+
 function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
@@ -18,4 +20,4 @@ function _nix() {
   _describe 'nix' suggestions
 }
 
-compdef _nix nix
+_nix "$@"

mk/lib.mk

@@ -10,8 +10,25 @@ bin-scripts :=
 noinst-scripts :=
 man-pages :=
 install-tests :=
-OS = $(shell uname -s)
 
+ifdef HOST_OS
+  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(HOST_KERNEL), cygwin)
+    HOST_CYGWIN = 1
+  endif
+  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
+    HOST_DARWIN = 1
+  endif
+  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
+    HOST_FREEBSD = 1
+  endif
+  ifeq ($(HOST_KERNEL), linux)
+    HOST_LINUX = 1
+  endif
+  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
+    HOST_SOLARIS = 1
+  endif
+endif
 
 # Hack to define a literal space.
 space :=
@@ -50,16 +67,16 @@ endif
 BUILD_SHARED_LIBS ?= 1
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
     GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
   else
     GLOBAL_CFLAGS += -fPIC
     GLOBAL_CXXFLAGS += -fPIC
   endif
-  ifneq ($(OS), Darwin)
-   ifneq ($(OS), SunOS)
-    ifneq ($(OS), FreeBSD)
+  ifndef HOST_DARWIN
+   ifndef HOST_SOLARIS
+    ifndef HOST_FREEBSD
      GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries
     endif
    endif

mk/libraries.mk

@@ -1,9 +1,9 @@
 libs-list :=
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
   SO_EXT = dylib
 else
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     SO_EXT = dll
   else
     SO_EXT = so
@@ -59,7 +59,7 @@ define build-library
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
 
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     $(1)_INSTALL_DIR ?= $$(bindir)
   else
     $(1)_INSTALL_DIR ?= $$(libdir)
@@ -73,18 +73,18 @@ define build-library
   ifeq ($(BUILD_SHARED_LIBS), 1)
 
     ifdef $(1)_ALLOW_UNDEFINED
-      ifeq ($(OS), Darwin)
+      ifdef HOST_DARWIN
         $(1)_LDFLAGS += -undefined suppress -flat_namespace
       endif
     else
-      ifneq ($(OS), Darwin)
-        ifneq (CYGWIN,$(findstring CYGWIN,$(OS)))
+      ifndef HOST_DARWIN
+        ifndef HOST_CYGWIN
           $(1)_LDFLAGS += -Wl,-z,defs
         endif
       endif
     endif
 
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT)
     endif
 
@@ -93,7 +93,7 @@ define build-library
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
 	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
     endif
     $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
@@ -108,7 +108,7 @@ define build-library
 	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       ifeq ($(SET_RPATH_TO_LIBS), 1)
         $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR)
       else

nix-rust/local.mk

@@ -11,12 +11,12 @@ libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
 libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust
 libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust
 
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 libnixrust_LDFLAGS_USE += -ldl
 libnixrust_LDFLAGS_USE_INSTALLED += -ldl
 endif
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
 else
 libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
@@ -31,7 +31,7 @@ $(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
 
 $(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
 	$(target-gen) cp $^ $@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 	install_name_tool -id $@ $@
 endif
 
@@ -40,7 +40,7 @@ clean: clean-rust
 clean-rust:
 	$(suppress) rm -rfv nix-rust/target
 
-ifneq ($(OS), Darwin)
+ifndef HOST_DARWIN
 check: rust-tests
 
 rust-tests:

perl/Makefile

@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -std=c++17
+GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src
 
 -include Makefile.config
 

perl/Makefile.config.in

@@ -1,3 +1,4 @@
+HOST_OS = @host_os@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@

perl/configure.ac

@@ -7,6 +7,8 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 
+AC_CANONICAL_HOST
+
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
 

perl/local.mk

@@ -28,7 +28,7 @@ Store_CXXFLAGS = \
 
 Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
 
-ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+ifdef HOST_CYGWIN
   archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
   libperl = $(shell perl -E 'use Config; print $$Config{libperl};')
   Store_LDFLAGS += $(shell find ${archlib} -name ${libperl})

scripts/install-nix-from-closure.sh

@@ -106,12 +106,11 @@ while [ $# -gt 0 ]; do
                 echo ""
                 echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
                 echo ""
-                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
-                echo "                      is installed by default."
+                echo " --no-modify-profile: Don't modify the user profile to automatically load nix."
                 echo ""
                 echo " --daemon-user-count: Number of build users to create. Defaults to 32."
                 echo ""
-                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
+                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix/nix.conf"
                 echo ""
                 if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then
                     echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from."

scripts/install.in

@@ -40,13 +40,23 @@ case "$(uname -s).$(uname -m)" in
         path=@tarballPath_aarch64-linux@
         system=aarch64-linux
         ;;
+    Linux.armv6l_linux)
+        hash=@tarballHash_armv6l-linux@
+        path=@tarballPath_armv6l-linux@
+        system=armv6l-linux
+        ;;
+    Linux.armv7l_linux)
+        hash=@tarballHash_armv7l-linux@
+        path=@tarballPath_armv7l-linux@
+        system=armv7l-linux
+        ;;
     Darwin.x86_64)
         hash=@tarballHash_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@
         system=x86_64-darwin
         ;;
     Darwin.arm64|Darwin.aarch64)
-        hash=@binaryTarball_aarch64-darwin@
+        hash=@tarballHash_aarch64-darwin@
         path=@tarballPath_aarch64-darwin@
         system=aarch64-darwin
         ;;

src/build-remote/build-remote.cc

@@ -277,7 +277,16 @@ connected:
 
         auto drv = store->readDerivation(*drvPath);
         auto outputHashes = staticOutputHashes(*store, drv);
-        drv.inputSrcs = store->parseStorePathSet(inputs);
+
+        // Hijack the inputs paths of the derivation to include all the paths
+        // that come from the `inputDrvs` set.
+        // We don’t do that for the derivations whose `inputDrvs` is empty
+        // because
+        // 1. It’s not needed
+        // 2. Changing the `inputSrcs` set changes the associated output ids,
+        //  which break CA derivations
+        if (!drv.inputDrvs.empty())
+            drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
 

src/libcmd/command.cc

@@ -188,7 +188,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
     }
 
     if (result.size() != 1)
-        throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
+        throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
     updateProfile(result[0]);
 }

src/libcmd/installables.cc

@@ -58,9 +58,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "no-registries",
-        .description = "Don't allow lookups in the flake registries.",
+        .description =
+          "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
         .category = category,
-        .handler = {&lockFlags.useRegistries, false}
+        .handler = {[&]() {
+            lockFlags.useRegistries = false;
+            warn("'--no-registries' is deprecated; use '--no-use-registries'");
+        }}
     });
 
     addFlag({
@@ -171,14 +175,50 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
-    if (file) return; // FIXME
+    if (file) {
+        evalSettings.pureEval = false;
+        auto state = getEvalState();
+        Expr *e = state->parseExprFromFile(
+            resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
+        );
 
-    completeFlakeRefWithFragment(
-        getEvalState(),
-        lockFlags,
-        getDefaultFlakeAttrPathPrefixes(),
-        getDefaultFlakeAttrPaths(),
-        prefix);
+        Value root;
+        state->eval(e, root);
+
+        auto autoArgs = getAutoArgs(*state);
+
+        std::string prefix_ = std::string(prefix);
+        auto sep = prefix_.rfind('.');
+        std::string searchWord;
+        if (sep != std::string::npos) {
+            searchWord = prefix_.substr(sep, std::string::npos);
+            prefix_ = prefix_.substr(0, sep);
+        } else {
+            searchWord = prefix_;
+            prefix_ = "";
+        }
+
+        Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first);
+        state->forceValue(v1);
+        Value v2;
+        state->autoCallFunction(*autoArgs, v1, v2);
+
+        if (v2.type() == nAttrs) {
+            for (auto & i : *v2.attrs) {
+                std::string name = i.name;
+                if (name.find(searchWord) == 0) {
+                    completions->add(i.name);
+                }
+            }
+        }
+    } else {
+        completeFlakeRefWithFragment(
+            getEvalState(),
+            lockFlags,
+            getDefaultFlakeAttrPathPrefixes(),
+            getDefaultFlakeAttrPaths(),
+            prefix);
+    }
 }
 
 void completeFlakeRefWithFragment(
@@ -573,10 +613,10 @@ InstallableFlake::getCursors(EvalState & state)
 
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
+    flake::LockFlags lockFlagsApplyConfig = lockFlags;
+    lockFlagsApplyConfig.applyNixConfig = true;
     if (!_lockedFlake) {
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
-        _lockedFlake->flake.config.apply();
-        // FIXME: send new config to the daemon.
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }

src/libexpr/attr-path.cc

@@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error("missing closing quote in selection path '%1%'", s);
+                    throw ParseError("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -100,7 +100,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 }
 
 
-Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
+Pos findPackageFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
@@ -116,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     auto colon = pos.rfind(':');
     if (colon == std::string::npos)
-        throw Error("cannot parse meta.position attribute '%s'", pos);
+        throw ParseError("cannot parse meta.position attribute '%s'", pos);
 
     std::string filename(pos, 0, colon);
     unsigned int lineno;
     try {
         lineno = std::stoi(std::string(pos, colon + 1));
     } catch (std::invalid_argument & e) {
-        throw Error("cannot parse line number '%s'", pos);
+        throw ParseError("cannot parse line number '%s'", pos);
     }
 
     Symbol file = state.symbols.create(filename);

src/libexpr/attr-path.hh

@@ -14,7 +14,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
-Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
+Pos findPackageFilename(EvalState & state, Value & v, std::string what);
 
 std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
 

src/libexpr/eval.cc

@@ -64,7 +64,11 @@ static char * dupStringWithLen(const char * s, size_t size)
 
 RootValue allocRootValue(Value * v)
 {
+#if HAVE_BOEHMGC
     return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
+#else
+    return std::make_shared<Value *>(v);
+#endif
 }
 
 
@@ -233,22 +237,34 @@ static void * oomHandler(size_t requested)
 }
 
 class BoehmGCStackAllocator : public StackAllocator {
-  boost::coroutines2::protected_fixedsize_stack stack {
-    // We allocate 8 MB, the default max stack size on NixOS.
-    // A smaller stack might be quicker to allocate but reduces the stack
-    // depth available for source filter expressions etc.
-    std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
+    boost::coroutines2::protected_fixedsize_stack stack {
+        // We allocate 8 MB, the default max stack size on NixOS.
+        // A smaller stack might be quicker to allocate but reduces the stack
+        // depth available for source filter expressions etc.
+        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
     };
 
+    // This is specific to boost::coroutines2::protected_fixedsize_stack.
+    // The stack protection page is included in sctx.size, so we have to
+    // subtract one page size from the stack size.
+    std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) {
+        return sctx.size - boost::context::stack_traits::page_size();
+    }
+
   public:
     boost::context::stack_context allocate() override {
         auto sctx = stack.allocate();
-        GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+
+        // Stacks generally start at a high address and grow to lower addresses.
+        // Architectures that do the opposite are rare; in fact so rare that
+        // boost_routine does not implement it.
+        // So we subtract the stack size.
+        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
         return sctx;
     }
 
     void deallocate(boost::context::stack_context sctx) override {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
         stack.deallocate(sctx);
     }
 
@@ -760,18 +776,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 }
 
 
-std::atomic<uint64_t> nrValuesFreed{0};
-
-void finalizeValue(void * obj, void * data)
-{
-    nrValuesFreed++;
-}
-
 Value * EvalState::allocValue()
 {
     nrValues++;
     auto v = (Value *) allocBytes(sizeof(Value));
-    //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
     return v;
 }
 
@@ -838,39 +846,37 @@ Value * Expr::maybeThunk(EvalState & state, Env & env)
 }
 
 
-unsigned long nrAvoided = 0;
-
 Value * ExprVar::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.lookupVar(&env, *this, true);
     /* The value might not be initialised in the environment yet.
        In that case, ignore it. */
-    if (v) { nrAvoided++; return v; }
+    if (v) { state.nrAvoided++; return v; }
     return Expr::maybeThunk(state, env);
 }
 
 
 Value * ExprString::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
 Value * ExprInt::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
 Value * ExprFloat::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
 Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
@@ -908,7 +914,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
         // computation.
         if (mustBeTrivial &&
             !(dynamic_cast<ExprAttrs *>(e)))
-            throw Error("file '%s' must be an attribute set", path);
+            throw EvalError("file '%s' must be an attribute set", path);
         eval(e, v);
     } catch (Error & e) {
         addErrorTrace(e, "while evaluating the file '%1%':", path2);
@@ -1125,8 +1131,6 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa
 }
 
 
-unsigned long nrLookups = 0;
-
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
@@ -1138,7 +1142,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
     try {
 
         for (auto & i : attrPath) {
-            nrLookups++;
+            state.nrLookups++;
             Bindings::iterator j;
             Symbol name = getName(i, state, env);
             if (def) {

src/libexpr/eval.hh

@@ -316,8 +316,10 @@ private:
     unsigned long nrValuesInEnvs = 0;
     unsigned long nrValues = 0;
     unsigned long nrListElems = 0;
+    unsigned long nrLookups = 0;
     unsigned long nrAttrsets = 0;
     unsigned long nrAttrsInAttrsets = 0;
+    unsigned long nrAvoided = 0;
     unsigned long nrOpUpdates = 0;
     unsigned long nrOpUpdateValuesCopied = 0;
     unsigned long nrListConcats = 0;
@@ -339,6 +341,11 @@ private:
 
     friend struct ExprOpUpdate;
     friend struct ExprOpConcatLists;
+    friend struct ExprVar;
+    friend struct ExprString;
+    friend struct ExprInt;
+    friend struct ExprFloat;
+    friend struct ExprPath;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);

src/libexpr/flake/flake.cc

@@ -1,4 +1,5 @@
 #include "flake.hh"
+#include "eval.hh"
 #include "lockfile.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
@@ -296,7 +297,14 @@ LockedFlake lockFlake(
 
     FlakeCache flakeCache;
 
-    auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
+    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
+
+    auto flake = getFlake(state, topRef, useRegistries, flakeCache);
+
+    if (lockFlags.applyNixConfig) {
+        flake.config.apply();
+        // FIXME: send new config to the daemon.
+    }
 
     try {
 
@@ -359,7 +367,12 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
-                    if (hasOverride) overridesUsed.insert(inputPath);
+                    if (hasOverride) {
+                        overridesUsed.insert(inputPath);
+                        // Respect the “flakeness” of the input even if we
+                        // override it
+                        i->second.isFlake = input2.isFlake;
+                    }
                     auto & input = hasOverride ? i->second : input2;
 
                     /* Resolve 'follows' later (since it may refer to an input
@@ -454,7 +467,7 @@ LockedFlake lockFlake(
                             throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                         if (input.isFlake) {
-                            auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
+                            auto inputFlake = getFlake(state, *input.ref, useRegistries, flakeCache);
 
                             /* Note: in case of an --override-input, we use
                                the *original* ref (input2.ref) for the
@@ -489,7 +502,7 @@ LockedFlake lockFlake(
 
                         else {
                             auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-                                state, *input.ref, lockFlags.useRegistries, flakeCache);
+                                state, *input.ref, useRegistries, flakeCache);
                             node->inputs.insert_or_assign(id,
                                 std::make_shared<LockedNode>(lockedRef, *input.ref, false));
                         }
@@ -563,7 +576,7 @@ LockedFlake lockFlake(
                            also just clear the 'rev' field... */
                         auto prevLockedRef = flake.lockedRef;
                         FlakeCache dummyCache;
-                        flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
+                        flake = getFlake(state, topRef, useRegistries, dummyCache);
 
                         if (lockFlags.commitLockFile &&
                             flake.lockedRef.input.getRev() &&
@@ -633,7 +646,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval,
+                .useRegistries = !evalSettings.pureEval && !settings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),
         v);

src/libexpr/flake/flake.hh

@@ -102,7 +102,11 @@ struct LockFlags
 
     /* Whether to use the registries to lookup indirect flake
        references like 'nixpkgs'. */
-    bool useRegistries = true;
+    std::optional<bool> useRegistries = std::nullopt;
+
+    /* Whether to apply flake's nixConfig attribute to the configuration */
+
+    bool applyNixConfig = false;
 
     /* Whether mutable flake references (i.e. those without a Git
        revision or similar) without a corresponding lock are

src/libexpr/lexer.l

@@ -38,11 +38,13 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
 
-    while (len--) {
+    for (size_t i = 0; i < len; i++) {
        switch (*s++) {
        case '\r':
-           if (*s == '\n') /* cr/lf */
+           if (*s == '\n') { /* cr/lf */
+               i++;
                s++;
+           }
            /* fall through */
        case '\n':
            ++loc->last_line;

src/libexpr/local.mk

@@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 libexpr_LIBS = libutil libstore libfetchers
 
 libexpr_LDFLAGS = -lboost_context
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
 

src/libexpr/primops.cc

@@ -3109,7 +3109,7 @@ static RegisterPrimOp primop_toString({
 
         - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`.
 
-        - A set containing `{ __toString = self: ...; }`.
+        - A set containing `{ __toString = self: ...; }` or `{ outPath = ...; }`.
 
         - An integer.
 

src/libexpr/primops/fetchMercurial.cc

@@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     fetchers::Attrs attrs;
     attrs.insert_or_assign("type", "hg");
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    attrs.insert_or_assign("name", name);
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(std::move(attrs));

src/libexpr/primops/fetchTree.cc

@@ -7,6 +7,7 @@
 
 #include <ctime>
 #include <iomanip>
+#include <regex>
 
 namespace nix {
 
@@ -60,10 +61,19 @@ void emitTreeAttrs(
     v.attrs->sort();
 }
 
-std::string fixURI(std::string uri, EvalState &state)
+std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
 {
     state.checkURI(uri);
-    return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+    return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
+}
+
+std::string fixURIForGit(std::string uri, EvalState & state)
+{
+    static std::regex scp_uri("([^/].*)@(.*):(.*)");
+    if (uri[0] != '/' && std::regex_match(uri, scp_uri))
+        return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
+    else
+        return fixURI(uri, state);
 }
 
 void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
@@ -72,13 +82,18 @@ void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v
     attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
 }
 
+struct FetchTreeParams {
+    bool emptyRevFallback = false;
+    bool allowNameArgument = false;
+};
+
 static void fetchTree(
     EvalState &state,
     const Pos &pos,
     Value **args,
     Value &v,
     const std::optional<std::string> type,
-    bool emptyRevFallback = false
+    const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
     PathSet context;
@@ -119,17 +134,25 @@ static void fetchTree(
                 .errPos = pos
             });
 
+        if (!params.allowNameArgument)
+            if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
+                throw Error({
+                    .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+                    .errPos = pos
+                });
+
+
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
-        auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+        auto url = state.coerceToString(pos, *args[0], context, false, false);
 
         if (type == "git") {
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
-            attrs.emplace("url", url);
+            attrs.emplace("url", fixURIForGit(url, state));
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
-            input = fetchers::Input::fromURL(url);
+            input = fetchers::Input::fromURL(fixURI(url, state));
         }
     }
 
@@ -144,13 +167,13 @@ static void fetchTree(
     if (state.allowedPaths)
         state.allowedPaths->insert(tree.actualPath);
 
-    emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
+    emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback);
 }
 
 static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     settings.requireExperimentalFeature("flakes");
-    fetchTree(state, pos, args, v, std::nullopt);
+    fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
 }
 
 // FIXME: document
@@ -292,7 +315,7 @@ static RegisterPrimOp primop_fetchTarball({
 
 static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
 {
-    fetchTree(state, pos, args, v, "git", true);
+    fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
 }
 
 static RegisterPrimOp primop_fetchGit({

src/libfetchers/fetchers.cc

@@ -200,12 +200,17 @@ void Input::markChangedFile(
     return scheme->markChangedFile(*this, file, commitMsg);
 }
 
+std::string Input::getName() const
+{
+    return maybeGetStrAttr(attrs, "name").value_or("source");
+}
+
 StorePath Input::computeStorePath(Store & store) const
 {
     auto narHash = getNarHash();
     if (!narHash)
         throw Error("cannot compute store path for mutable input '%s'", to_string());
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
 }
 
 std::string Input::getType() const

src/libfetchers/fetchers.hh

@@ -81,6 +81,8 @@ public:
         std::string_view file,
         std::optional<std::string> commitMsg) const;
 
+    std::string getName() const;
+
     StorePath computeStorePath(Store & store) const;
 
     // Convenience functions for common attributes.

src/libfetchers/git.cc

@@ -60,7 +60,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -167,10 +167,10 @@ struct GitInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        std::string name = input.getName();
+
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@@ -270,7 +270,7 @@ struct GitInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 // FIXME: maybe we should use the timestamp of the last
                 // modified dirty file?

src/libfetchers/github.cc

@@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
         input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 

src/libfetchers/mercurial.cc

@@ -74,7 +74,7 @@ struct MercurialInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "hg") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
                 throw Error("unsupported Mercurial input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -147,10 +147,10 @@ struct MercurialInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        auto name = input.getName();
+
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
@@ -193,7 +193,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {
                     Tree(store->toRealPath(storePath), std::move(storePath)),

src/libfetchers/registry.cc

@@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
     return userRegistry;
 }
 
+std::shared_ptr<Registry> getCustomRegistry(const Path & p)
+{
+    static auto customRegistry =
+        Registry::read(p, Registry::Custom);
+    return customRegistry;
+}
+
 static std::shared_ptr<Registry> flagRegistry =
     std::make_shared<Registry>(Registry::Flag);
 

src/libfetchers/registry.hh

@@ -14,6 +14,7 @@ struct Registry
         User = 1,
         System = 2,
         Global = 3,
+        Custom = 4,
     };
 
     RegistryType type;
@@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
 
 std::shared_ptr<Registry> getUserRegistry();
 
+std::shared_ptr<Registry> getCustomRegistry(const Path & p);
+
 Path getUserRegistryPath();
 
 Registries getRegistries(ref<Store> store);

src/libfetchers/tarball.cc

@@ -196,7 +196,7 @@ struct TarballInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
+            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         Input input;
@@ -226,7 +226,7 @@ struct TarballInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
         return {std::move(tree), input};
     }
 };

src/libmain/progress-bar.cc

@@ -484,7 +484,7 @@ Logger * makeProgressBar(bool printBuildLogs)
 {
     return new ProgressBar(
         printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+        shouldANSI()
     );
 }
 

src/libmain/shared.cc

@@ -238,7 +238,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     addFlag({
         .longName = "no-gc-warning",
         .description = "Disable warnings about not using `--add-root`.",
-        .handler = {&gcWarning, true},
+        .handler = {&gcWarning, false},
     });
 
     addFlag({

src/libstore/build/derivation-goal.cc

@@ -143,7 +143,6 @@ void DerivationGoal::work()
     (this->*state)();
 }
 
-
 void DerivationGoal::addWantedOutputs(const StringSet & outputs)
 {
     /* If we already want all outputs, there is nothing to do. */
@@ -545,7 +544,7 @@ void DerivationGoal::tryToBuild()
     PathSet lockFiles;
     /* FIXME: Should lock something like the drv itself so we don't build same
        CA drv concurrently */
-    if (dynamic_cast<LocalStore *>(&worker.store))
+    if (dynamic_cast<LocalStore *>(&worker.store)) {
         /* If we aren't a local store, we might need to use the local store as
            a build remote, but that would cause a deadlock. */
         /* FIXME: Make it so we can use ourselves as a build remote even if we
@@ -553,9 +552,15 @@ void DerivationGoal::tryToBuild()
         /* FIXME: find some way to lock for scheduling for the other stores so
            a forking daemon with --store still won't farm out redundant builds.
            */
-        for (auto & i : drv->outputsAndOptPaths(worker.store))
+        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
             if (i.second.second)
                 lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+            else
+                lockFiles.insert(
+                    worker.store.Store::toRealPath(drvPath) + "!" + i.first
+                );
+        }
+    }
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         if (!actLock)
@@ -739,6 +744,64 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 {
 }
 
+void runPostBuildHook(
+    Store & store,
+    Logger & logger,
+    const StorePath & drvPath,
+    StorePathSet outputPaths
+)
+{
+    auto hook = settings.postBuildHook;
+    if (hook == "")
+        return;
+
+    Activity act(logger, lvlInfo, actPostBuildHook,
+            fmt("running post-build-hook '%s'", settings.postBuildHook),
+            Logger::Fields{store.printStorePath(drvPath)});
+    PushActivity pact(act.id);
+    std::map<std::string, std::string> hookEnvironment = getEnv();
+
+    hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
+    hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
+    hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
+
+    RunOptions opts(settings.postBuildHook, {});
+    opts.environment = hookEnvironment;
+
+    struct LogSink : Sink {
+        Activity & act;
+        std::string currentLine;
+
+        LogSink(Activity & act) : act(act) { }
+
+        void operator() (std::string_view data) override {
+            for (auto c : data) {
+                if (c == '\n') {
+                    flushLine();
+                } else {
+                    currentLine += c;
+                }
+            }
+        }
+
+        void flushLine() {
+            act.result(resPostBuildLogLine, currentLine);
+            currentLine.clear();
+        }
+
+        ~LogSink() {
+            if (currentLine != "") {
+                currentLine += '\n';
+                flushLine();
+            }
+        }
+    };
+    LogSink sink(act);
+
+    opts.standardOut = &sink;
+    opts.mergeStderrToStdout = true;
+    runProgram2(opts);
+}
 
 void DerivationGoal::buildDone()
 {
@@ -804,57 +867,15 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (std::string_view data) override {
-                    for (auto c : data) {
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
+        StorePathSet outputPaths;
+        for (auto & [_, path] : finalOutputs)
+            outputPaths.insert(path);
+        runPostBuildHook(
+            worker.store,
+            *logger,
+            drvPath,
+            outputPaths
+        );
 
         if (buildMode == bmCheck) {
             cleanupPostOutputsRegisteredModeCheck();
@@ -910,6 +931,8 @@ void DerivationGoal::resolvedFinished() {
 
     auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
 
+    StorePathSet outputPaths;
+
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
@@ -927,8 +950,10 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
+            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
+            outputPaths.insert(realisation->outPath);
         } else {
             // If we don't have a realisation, then it must mean that something
             // failed when building the resolved drv
@@ -936,6 +961,13 @@ void DerivationGoal::resolvedFinished() {
         }
     }
 
+    runPostBuildHook(
+        worker.store,
+        *logger,
+        drvPath,
+        outputPaths
+    );
+
     // This is potentially a bit fishy in terms of error reporting. Not sure
     // how to do it in a cleaner way
     amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
@@ -1048,42 +1080,6 @@ HookReply DerivationGoal::tryBuildHook()
 }
 
 
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output

src/libstore/build/drv-output-substitution-goal.cc

@@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
 void DrvOutputSubstitutionGoal::init()
 {
     trace("init");
+
+    /* If the derivation already exists, we’re done */
+    if (worker.store.queryRealisation(id)) {
+        amDone(ecSuccess);
+        return;
+    }
+
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
     tryNext();
 }
@@ -53,6 +60,26 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
+    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+        if (depId != id) {
+            if (auto localOutputInfo = worker.store.queryRealisation(depId);
+                localOutputInfo && localOutputInfo->outPath != depPath) {
+                warn(
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                    "Local:  %s\n"
+                    "Remote: %s",
+                    sub->getUri(),
+                    depId.to_string(),
+                    worker.store.printStorePath(localOutputInfo->outPath),
+                    worker.store.printStorePath(depPath)
+                );
+                tryNext();
+                return;
+            }
+            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+        }
+    }
+
     addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
     if (waitees.empty()) outPathValid();

src/libstore/build/local-derivation-goal.cc

@@ -292,7 +292,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         auto & localStore = getLocalStore();
         uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
         struct statvfs st;
-        if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
+        if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
             (uint64_t) st.f_bavail * st.f_bsize < required)
             diskFull = true;
         if (statvfs(tmpDir.c_str(), &st) == 0 &&
@@ -417,7 +417,7 @@ void LocalDerivationGoal::startBuilder()
     }
 
     auto & localStore = getLocalStore();
-    if (localStore.storeDir != localStore.realStoreDir) {
+    if (localStore.storeDir != localStore.realStoreDir.get()) {
         #if __linux__
             useChroot = true;
         #else
@@ -518,7 +518,7 @@ void LocalDerivationGoal::startBuilder()
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
                 worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
+                    worker.store.exportReferences({storePath}, inputPaths), false, false));
         }
     }
 
@@ -1084,113 +1084,28 @@ void LocalDerivationGoal::initEnv()
 }
 
 
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
 void LocalDerivationGoal::writeStructuredAttrs()
 {
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
+    if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) {
+        auto json = structAttrsJson.value();
+        nlohmann::json rewritten;
+        for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
+            /* The placeholder must have a rewrite, so we use it to cover both the
+               cases where we know or don't know the output path ahead of time. */
+            rewritten[i] = rewriteStrings(v, inputRewrites);
+        }
 
-    auto json = *structuredAttrs;
+        json["outputs"] = rewritten;
 
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
+        auto jsonSh = writeStructuredAttrsShell(json);
+
+        writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
+        chownToBuilder(tmpDir + "/.attrs.sh");
+        env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh";
+        writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
+        chownToBuilder(tmpDir + "/.attrs.json");
+        env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json";
     }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
 
@@ -1333,13 +1248,18 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
     // XXX: This should probably be allowed if the realisation corresponds to
     // an allowed derivation
-    { throw Error("queryRealisation"); }
+    {
+        if (!goal.isAllowed(id))
+            throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
+        return next->queryRealisation(id);
+    }
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
         StorePathSet newPaths;
+        std::set<Realisation> newRealisations;
 
         for (auto & req : paths) {
             if (!goal.isAllowed(req))
@@ -1352,16 +1272,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             auto p =  std::get_if<DerivedPath::Built>(&path);
             if (!p) continue;
             auto & bfd = *p;
+            auto drv = readDerivation(bfd.drvPath);
+            auto drvHashes = staticOutputHashes(*this, drv);
             auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
             for (auto & [outputName, outputPath] : outputs)
-                if (wantOutput(outputName, bfd.outputs))
+                if (wantOutput(outputName, bfd.outputs)) {
                     newPaths.insert(outputPath);
+                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                        auto thisRealisation = next->queryRealisation(
+                            DrvOutput{drvHashes.at(outputName), outputName}
+                        );
+                        assert(thisRealisation);
+                        newRealisations.insert(*thisRealisation);
+                    }
+                }
         }
 
         StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
             goal.addDependency(path);
+        for (auto & real : Realisation::closure(*next, newRealisations))
+            goal.addedDrvOutputs.insert(real.id);
     }
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -1737,7 +1669,7 @@ void LocalDerivationGoal::runChild()
                 /* N.B. it is realistic that these paths might not exist. It
                    happens when testing Nix building fixed-output derivations
                    within a pure derivation. */
-                for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
+                for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
                     if (pathExists(path))
                         ss.push_back(path);
             }
@@ -2464,6 +2396,7 @@ void LocalDerivationGoal::registerOutputs()
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         PathLocks dynamicOutputLock;
+        dynamicOutputLock.setDeletion(true);
         auto optFixedPath = output.path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)
@@ -2487,6 +2420,7 @@ void LocalDerivationGoal::registerOutputs()
                 assert(newInfo.ca);
             } else {
                 auto destPath = worker.store.toRealPath(finalDestPath);
+                deletePath(destPath);
                 movePath(actualPath, destPath);
                 actualPath = destPath;
             }

src/libstore/build/local-derivation-goal.hh

@@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Paths that were added via recursive Nix calls. */
     StorePathSet addedPaths;
 
+    /* Realisations that were added via recursive Nix calls. */
+    std::set<DrvOutput> addedDrvOutputs;
+
     /* Recursive Nix calls are only allowed to build or realize paths
        in the original input closure or added via a recursive Nix call
        (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
+    bool isAllowed(const DrvOutput & id)
+    {
+        return addedDrvOutputs.count(id);
+    }
+
     bool isAllowed(const DerivedPath & req);
 
     friend struct RestrictedStore;

src/libstore/ca-specific-schema.sql

@@ -3,10 +3,19 @@
 -- is enabled
 
 create table if not exists Realisations (
+    id integer primary key autoincrement not null,
     drvPath text not null,
     outputName text not null, -- symbolic output id, usually "out"
     outputPath integer not null,
     signatures text, -- space-separated list
-    primary key (drvPath, outputName),
     foreign key (outputPath) references ValidPaths(id) on delete cascade
 );
+
+create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+create table if not exists RealisationsRefs (
+    referrer integer not null,
+    realisationReference integer,
+    foreign key (referrer) references Realisations(id) on delete cascade,
+    foreign key (realisationReference) references Realisations(id) on delete restrict
+);

src/libstore/daemon.cc

@@ -885,10 +885,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopRegisterDrvOutput: {
         logger->startWork();
-        auto outputId = DrvOutput::parse(readString(from));
-        auto outputPath = StorePath(readString(from));
-        store->registerDrvOutput(Realisation{
-            .id = outputId, .outPath = outputPath});
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            auto outputId = DrvOutput::parse(readString(from));
+            auto outputPath = StorePath(readString(from));
+            store->registerDrvOutput(Realisation{
+                .id = outputId, .outPath = outputPath});
+        } else {
+            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
+            store->registerDrvOutput(realisation);
+        }
         logger->stopWork();
         break;
     }
@@ -898,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto outputId = DrvOutput::parse(readString(from));
         auto info = store->queryRealisation(outputId);
         logger->stopWork();
-        std::set<StorePath> outPaths;
-        if (info) outPaths.insert(info->outPath);
-        worker_proto::write(*store, to, outPaths);
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            std::set<StorePath> outPaths;
+            if (info) outPaths.insert(info->outPath);
+            worker_proto::write(*store, to, outPaths);
+        } else {
+            std::set<Realisation> realisations;
+            if (info) realisations.insert(*info);
+            worker_proto::write(*store, to, realisations);
+        }
         break;
     }
 

src/libstore/gc.cc

@@ -775,7 +775,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         try {
 
-            AutoCloseDir dir(opendir(realStoreDir.c_str()));
+            AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
             if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
             /* Read the store and immediately delete all paths that
@@ -856,7 +856,7 @@ void LocalStore::autoGC(bool sync)
             return std::stoll(readFile(*fakeFreeSpaceFile));
 
         struct statvfs st;
-        if (statvfs(realStoreDir.c_str(), &st))
+        if (statvfs(realStoreDir.get().c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
 
         return (uint64_t) st.f_bavail * st.f_frsize;

src/libstore/globals.hh

@@ -956,6 +956,9 @@ public:
           resolves to a different location from that of the build machine. You
           can enable this setting if you are sure you're not going to do that.
         )"};
+
+    Setting<bool> useRegistries{this, true, "use-registries",
+        "Whether to use flake registries to resolve flake references."};
 };
 
 

src/libstore/local-fs-store.hh

@@ -18,6 +18,9 @@ struct LocalFSStoreConfig : virtual StoreConfig
     const PathSetting logDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
         "log", "directory where Nix will store state"};
+    const PathSetting realStoreDir{(StoreConfig*) this, false,
+        rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
+        "physical path to the Nix store"};
 };
 
 class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
@@ -34,7 +37,7 @@ public:
     /* Register a permanent GC root. */
     Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
 
-    virtual Path getRealStoreDir() { return storeDir; }
+    virtual Path getRealStoreDir() { return realStoreDir; }
 
     Path toRealPath(const Path & storePath) override
     {

src/libstore/local-store.cc

@@ -53,12 +53,15 @@ struct LocalStore::State::Stmts {
     SQLiteStmt InvalidatePath;
     SQLiteStmt AddDerivationOutput;
     SQLiteStmt RegisterRealisedOutput;
+    SQLiteStmt UpdateRealisedOutput;
     SQLiteStmt QueryValidDerivers;
     SQLiteStmt QueryDerivationOutputs;
     SQLiteStmt QueryRealisedOutput;
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
+    SQLiteStmt QueryRealisationReferences;
+    SQLiteStmt AddRealisationReference;
 };
 
 int getSchema(Path schemaPath)
@@ -76,7 +79,7 @@ int getSchema(Path schemaPath)
 
 void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 {
-    const int nixCASchemaVersion = 1;
+    const int nixCASchemaVersion = 2;
     int curCASchema = getSchema(schemaPath);
     if (curCASchema != nixCASchemaVersion) {
         if (curCASchema > nixCASchemaVersion) {
@@ -94,7 +97,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
               #include "ca-specific-schema.sql.gen.hh"
                 ;
             db.exec(schema);
+            curCASchema = nixCASchemaVersion;
         }
+
+        if (curCASchema < 2) {
+            SQLiteTxn txn(db);
+            // Ugly little sql dance to add a new `id` column and make it the primary key
+            db.exec(R"(
+                create table Realisations2 (
+                    id integer primary key autoincrement not null,
+                    drvPath text not null,
+                    outputName text not null, -- symbolic output id, usually "out"
+                    outputPath integer not null,
+                    signatures text, -- space-separated list
+                    foreign key (outputPath) references ValidPaths(id) on delete cascade
+                );
+                insert into Realisations2 (drvPath, outputName, outputPath, signatures)
+                    select drvPath, outputName, outputPath, signatures from Realisations;
+                drop table Realisations;
+                alter table Realisations2 rename to Realisations;
+            )");
+            db.exec(R"(
+                create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+                create table if not exists RealisationsRefs (
+                    referrer integer not null,
+                    realisationReference integer,
+                    foreign key (referrer) references Realisations(id) on delete cascade,
+                    foreign key (realisationReference) references Realisations(id) on delete restrict
+                );
+            )");
+            txn.commit();
+        }
+
         writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
         lockFile(lockFd.get(), ltRead, true);
     }
@@ -106,9 +141,6 @@ LocalStore::LocalStore(const Params & params)
     , LocalStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
-    , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
-        "physical path to the Nix store"}
-    , realStoreDir(realStoreDir_)
     , dbDir(stateDir + "/db")
     , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
@@ -153,13 +185,13 @@ LocalStore::LocalStore(const Params & params)
             printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
         else {
             struct stat st;
-            if (stat(realStoreDir.c_str(), &st))
+            if (stat(realStoreDir.get().c_str(), &st))
                 throw SysError("getting attributes of path '%1%'", realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
-                if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
+                if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
                     throw SysError("changing ownership of path '%1%'", realStoreDir);
-                if (chmod(realStoreDir.c_str(), perm) == -1)
+                if (chmod(realStoreDir.get().c_str(), perm) == -1)
                     throw SysError("changing permissions on path '%1%'", realStoreDir);
             }
         }
@@ -310,13 +342,22 @@ LocalStore::LocalStore(const Params & params)
     if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
         state->stmts->RegisterRealisedOutput.create(state->db,
             R"(
-                insert or replace into Realisations (drvPath, outputName, outputPath, signatures)
+                insert into Realisations (drvPath, outputName, outputPath, signatures)
                 values (?, ?, (select id from ValidPaths where path = ?), ?)
                 ;
             )");
+        state->stmts->UpdateRealisedOutput.create(state->db,
+            R"(
+                update Realisations
+                    set signatures = ?
+                where
+                    drvPath = ? and
+                    outputName = ?
+                ;
+            )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
-                select Output.path, Realisations.signatures from Realisations
+                select Realisations.id, Output.path, Realisations.signatures from Realisations
                     inner join ValidPaths as Output on Output.id = Realisations.outputPath
                     where drvPath = ? and outputName = ?
                     ;
@@ -328,6 +369,19 @@ LocalStore::LocalStore(const Params & params)
                     where drvPath = ?
                     ;
             )");
+        state->stmts->QueryRealisationReferences.create(state->db,
+            R"(
+                select drvPath, outputName from Realisations
+                    join RealisationsRefs on realisationReference = Realisations.id
+                    where referrer = ?;
+            )");
+        state->stmts->AddRealisationReference.create(state->db,
+            R"(
+                insert or replace into RealisationsRefs (referrer, realisationReference)
+                values (
+                    (select id from Realisations where drvPath = ? and outputName = ?),
+                    (select id from Realisations where drvPath = ? and outputName = ?));
+            )");
     }
 }
 
@@ -437,14 +491,14 @@ void LocalStore::makeStoreWritable()
     if (getuid() != 0) return;
     /* Check if /nix/store is on a read-only mount. */
     struct statvfs stat;
-    if (statvfs(realStoreDir.c_str(), &stat) != 0)
+    if (statvfs(realStoreDir.get().c_str(), &stat) != 0)
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
-        if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
+        if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
             throw SysError("remounting %1% writable", realStoreDir);
     }
 #endif
@@ -664,14 +718,54 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
     settings.requireExperimentalFeature("ca-derivations");
-    auto state(_state.lock());
     retrySQLite<void>([&]() {
-        state->stmts->RegisterRealisedOutput.use()
-            (info.id.strHash())
-            (info.id.outputName)
-            (printStorePath(info.outPath))
-            (concatStringsSep(" ", info.signatures))
-            .exec();
+        auto state(_state.lock());
+        if (auto oldR = queryRealisation_(*state, info.id)) {
+            if (info.isCompatibleWith(*oldR)) {
+                auto combinedSignatures = oldR->signatures;
+                combinedSignatures.insert(info.signatures.begin(),
+                    info.signatures.end());
+                state->stmts->UpdateRealisedOutput.use()
+                    (concatStringsSep(" ", combinedSignatures))
+                    (info.id.strHash())
+                    (info.id.outputName)
+                    .exec();
+            } else {
+                throw Error("Trying to register a realisation of '%s', but we already "
+                            "have another one locally.\n"
+                            "Local:  %s\n"
+                            "Remote: %s",
+                    info.id.to_string(),
+                    printStorePath(oldR->outPath),
+                    printStorePath(info.outPath)
+                );
+            }
+        } else {
+            state->stmts->RegisterRealisedOutput.use()
+                (info.id.strHash())
+                (info.id.outputName)
+                (printStorePath(info.outPath))
+                (concatStringsSep(" ", info.signatures))
+                .exec();
+        }
+        for (auto & [outputId, depPath] : info.dependentRealisations) {
+            auto localRealisation = queryRealisationCore_(*state, outputId);
+            if (!localRealisation)
+                throw Error("unable to register the derivation '%s' as it "
+                            "depends on the non existent '%s'",
+                    info.id.to_string(), outputId.to_string());
+            if (localRealisation->second.outPath != depPath)
+                throw Error("unable to register the derivation '%s' as it "
+                            "depends on a realisation of '%s' that doesn’t"
+                            "match what we have locally",
+                    info.id.to_string(), outputId.to_string());
+            state->stmts->AddRealisationReference.use()
+                (info.id.strHash())
+                (info.id.outputName)
+                (outputId.strHash())
+                (outputId.outputName)
+                .exec();
+        }
     });
 }
 
@@ -1682,22 +1776,68 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
     }
 }
 
-std::optional<const Realisation> LocalStore::queryRealisation(
-    const DrvOutput& id) {
-    typedef std::optional<const Realisation> Ret;
-    return retrySQLite<Ret>([&]() -> Ret {
-        auto state(_state.lock());
-        auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
-            id.outputName));
-        if (!use.next())
-            return std::nullopt;
-        auto outputPath = parseStorePath(use.getStr(0));
-        auto signatures = tokenizeString<StringSet>(use.getStr(1));
-        return Ret{Realisation{
-            .id = id, .outPath = outputPath, .signatures = signatures}};
-    });
+std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_(
+        LocalStore::State & state,
+        const DrvOutput & id)
+{
+    auto useQueryRealisedOutput(
+            state.stmts->QueryRealisedOutput.use()
+                (id.strHash())
+                (id.outputName));
+    if (!useQueryRealisedOutput.next())
+        return std::nullopt;
+    auto realisationDbId = useQueryRealisedOutput.getInt(0);
+    auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
+    auto signatures =
+        tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
+
+    return {{
+        realisationDbId,
+        Realisation{
+            .id = id,
+            .outPath = outputPath,
+            .signatures = signatures,
+        }
+    }};
 }
 
+std::optional<const Realisation> LocalStore::queryRealisation_(
+            LocalStore::State & state,
+            const DrvOutput & id)
+{
+    auto maybeCore = queryRealisationCore_(state, id);
+    if (!maybeCore)
+        return std::nullopt;
+    auto [realisationDbId, res] = *maybeCore;
+
+    std::map<DrvOutput, StorePath> dependentRealisations;
+    auto useRealisationRefs(
+        state.stmts->QueryRealisationReferences.use()
+            (realisationDbId));
+    while (useRealisationRefs.next()) {
+        auto depId = DrvOutput {
+            Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
+            useRealisationRefs.getStr(1),
+        };
+        auto dependentRealisation = queryRealisationCore_(state, depId);
+        assert(dependentRealisation); // Enforced by the db schema
+        auto outputPath = dependentRealisation->second.outPath;
+        dependentRealisations.insert({depId, outputPath});
+    }
+
+    res.dependentRealisations = dependentRealisations;
+
+    return { res };
+}
+
+std::optional<const Realisation>
+LocalStore::queryRealisation(const DrvOutput & id)
+{
+    return retrySQLite<std::optional<const Realisation>>([&]() {
+        auto state(_state.lock());
+        return queryRealisation_(*state, id);
+    });
+}
 
 FixedOutputHash LocalStore::hashCAPath(
     const FileIngestionMethod & method, const HashType & hashType,

src/libstore/local-store.hh

@@ -83,9 +83,6 @@ private:
 
 public:
 
-    PathSetting realStoreDir_;
-
-    const Path realStoreDir;
     const Path dbDir;
     const Path linksDir;
     const Path reservedPath;
@@ -206,6 +203,8 @@ public:
     void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
     void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
 
+    std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
+    std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
 
 private:
@@ -279,8 +278,6 @@ private:
     void signPathInfo(ValidPathInfo & info);
     void signRealisation(Realisation &);
 
-    Path getRealStoreDir() override { return realStoreDir; }
-
     void createUser(const std::string & userName, uid_t userId) override;
 
     // XXX: Make a generic `Store` method

src/libstore/local.mk

@@ -9,11 +9,11 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 libstore_LIBS = libutil
 
 libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
 endif
 
@@ -23,7 +23,7 @@ ifeq ($(ENABLE_S3), 1)
 	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
 endif
 
-ifeq ($(OS), SunOS)
+ifdef HOST_SOLARIS
 	libstore_LDFLAGS += -lsocket
 endif
 

src/libstore/machines.cc

@@ -16,13 +16,18 @@ Machine::Machine(decltype(storeUri) storeUri,
     decltype(mandatoryFeatures) mandatoryFeatures,
     decltype(sshPublicHostKey) sshPublicHostKey) :
     storeUri(
-        // Backwards compatibility: if the URI is a hostname,
-        // prepend ssh://.
+        // Backwards compatibility: if the URI is schemeless, is not a path,
+        // and is not one of the special store connection words, prepend
+        // ssh://.
         storeUri.find("://") != std::string::npos
-        || hasPrefix(storeUri, "local")
-        || hasPrefix(storeUri, "remote")
-        || hasPrefix(storeUri, "auto")
-        || hasPrefix(storeUri, "/")
+        || storeUri.find("/") != std::string::npos
+        || storeUri == "auto"
+        || storeUri == "daemon"
+        || storeUri == "local"
+        || hasPrefix(storeUri, "auto?")
+        || hasPrefix(storeUri, "daemon?")
+        || hasPrefix(storeUri, "local?")
+        || hasPrefix(storeUri, "?")
         ? storeUri
         : "ssh://" + storeUri),
     systemTypes(systemTypes),

src/libstore/misc.cc

@@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(i);
 
             if (includeDerivers && path.isDerivation())
-                for (auto& i : queryDerivationOutputs(path))
-                    if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                        res.insert(i);
+                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
+                    if (maybeOutPath && isValidPath(*maybeOutPath))
+                        res.insert(*maybeOutPath);
             return res;
         };
     else
@@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(ref);
 
             if (includeOutputs && path.isDerivation())
-                for (auto& i : queryDerivationOutputs(path))
-                    if (isValidPath(i))
-                        res.insert(i);
+                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
+                    if (maybeOutPath && isValidPath(*maybeOutPath))
+                        res.insert(*maybeOutPath);
 
             if (includeDerivers && info->deriver && isValidPath(*info->deriver))
                 res.insert(*info->deriver);
@@ -254,5 +254,44 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         }});
 }
 
+std::map<DrvOutput, StorePath> drvOutputReferences(
+    const std::set<Realisation> & inputRealisations,
+    const StorePathSet & pathReferences)
+{
+    std::map<DrvOutput, StorePath> res;
 
+    for (const auto & input : inputRealisations) {
+        if (pathReferences.count(input.outPath)) {
+            res.insert({input.id, input.outPath});
+        }
+    }
+
+    return res;
+}
+
+std::map<DrvOutput, StorePath> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath)
+{
+    std::set<Realisation> inputRealisations;
+
+    for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
+        auto outputHashes =
+            staticOutputHashes(store, store.readDerivation(inputDrv));
+        for (const auto& outputName : outputNames) {
+            auto thisRealisation = store.queryRealisation(
+                DrvOutput{outputHashes.at(outputName), outputName});
+            if (!thisRealisation)
+                throw Error(
+                    "output '%s' of derivation '%s' isn’t built", outputName,
+                    store.printStorePath(inputDrv));
+            inputRealisations.insert(*thisRealisation);
+        }
+    }
+
+    auto info = store.queryPathInfo(outputPath);
+
+    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
+}
 }

src/libstore/optimise-store.cc

@@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
        permissions). */
-    bool mustToggle = dirOf(path) != realStoreDir;
+    bool mustToggle = dirOf(path) != realStoreDir.get();
     if (mustToggle) makeWritable(dirOf(path));
 
     /* When we're done, make the directory read-only again and reset

src/libstore/parsed-derivations.cc

@@ -1,6 +1,8 @@
 #include "parsed-derivations.hh"
 
 #include <nlohmann/json.hpp>
+#include <regex>
+#include "json.hh"
 
 namespace nix {
 
@@ -91,6 +93,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
     StringSet res;
     for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
         res.insert(i);
+    if (!derivationHasKnownOutputPaths(drv.type()))
+        res.insert("ca-derivations");
     return res;
 }
 
@@ -121,4 +125,107 @@ bool ParsedDerivation::substitutesAllowed() const
     return getBoolAttr("allowSubstitutes", true);
 }
 
+static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
+
+std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
+{
+    auto structuredAttrs = getStructuredAttrs();
+    if (!structuredAttrs) return std::nullopt;
+
+    auto json = *structuredAttrs;
+
+    /* Add an "outputs" object containing the output paths. */
+    nlohmann::json outputs;
+    for (auto & i : drv.outputs)
+        outputs[i.first] = hashPlaceholder(i.first);
+    json["outputs"] = outputs;
+
+    /* Handle exportReferencesGraph. */
+    auto e = json.find("exportReferencesGraph");
+    if (e != json.end() && e->is_object()) {
+        for (auto i = e->begin(); i != e->end(); ++i) {
+            std::ostringstream str;
+            {
+                JSONPlaceholder jsonRoot(str, true);
+                StorePathSet storePaths;
+                for (auto & p : *i)
+                    storePaths.insert(store.parseStorePath(p.get<std::string>()));
+                store.pathInfoToJSON(jsonRoot,
+                    store.exportReferences(storePaths, inputPaths), false, true);
+            }
+            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+        }
+    }
+
+    return json;
+}
+
+/* As a convenience to bash scripts, write a shell file that
+   maps all attributes that are representable in bash -
+   namely, strings, integers, nulls, Booleans, and arrays and
+   objects consisting entirely of those values. (So nested
+   arrays or objects are not supported.) */
+std::string writeStructuredAttrsShell(const nlohmann::json & json)
+{
+
+    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
+        if (value.is_string())
+            return shellEscape(value);
+
+        if (value.is_number()) {
+            auto f = value.get<float>();
+            if (std::ceil(f) == f)
+                return std::to_string(value.get<int>());
+        }
+
+        if (value.is_null())
+            return std::string("''");
+
+        if (value.is_boolean())
+            return value.get<bool>() ? std::string("1") : std::string("");
+
+        return {};
+    };
+
+    std::string jsonSh;
+
+    for (auto & [key, value] : json.items()) {
+
+        if (!std::regex_match(key, shVarName)) continue;
+
+        auto s = handleSimpleType(value);
+        if (s)
+            jsonSh += fmt("declare %s=%s\n", key, *s);
+
+        else if (value.is_array()) {
+            std::string s2;
+            bool good = true;
+
+            for (auto & value2 : value) {
+                auto s3 = handleSimpleType(value2);
+                if (!s3) { good = false; break; }
+                s2 += *s3; s2 += ' ';
+            }
+
+            if (good)
+                jsonSh += fmt("declare -a %s=(%s)\n", key, s2);
+        }
+
+        else if (value.is_object()) {
+            std::string s2;
+            bool good = true;
+
+            for (auto & [key2, value2] : value.items()) {
+                auto s3 = handleSimpleType(value2);
+                if (!s3) { good = false; break; }
+                s2 += fmt("[%s]=%s ", shellEscape(key2), *s3);
+            }
+
+            if (good)
+                jsonSh += fmt("declare -A %s=(%s)\n", key, s2);
+        }
+    }
+
+    return jsonSh;
+}
 }

src/libstore/parsed-derivations.hh

@@ -36,6 +36,10 @@ public:
     bool willBuildLocally(Store & localStore) const;
 
     bool substitutesAllowed() const;
+
+    std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths);
 };
 
+std::string writeStructuredAttrsShell(const nlohmann::json & json);
+
 }

src/libstore/profiles.cc

@@ -211,12 +211,15 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
 
 void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
 {
+    if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
+        throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
+
     time_t curTime = time(0);
     string strDays = string(timeSpec, 0, timeSpec.size() - 1);
     auto days = string2Int<int>(strDays);
 
     if (!days || *days < 1)
-        throw Error("invalid number of days specifier '%1%'", timeSpec);
+        throw UsageError("invalid number of days specifier '%1%'", timeSpec);
 
     time_t oldTime = curTime - *days * 24 * 3600;
 

src/libstore/realisation.cc

@@ -1,5 +1,6 @@
 #include "realisation.hh"
 #include "store-api.hh"
+#include "closure.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -21,11 +22,52 @@ std::string DrvOutput::to_string() const {
     return strHash() + "!" + outputName;
 }
 
+std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
+{
+    std::set<Realisation> res;
+    Realisation::closure(store, startOutputs, res);
+    return res;
+}
+
+void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
+{
+    auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
+        std::set<Realisation> res;
+        for (auto& [currentDep, _] : current.dependentRealisations) {
+            if (auto currentRealisation = store.queryRealisation(currentDep))
+                res.insert(*currentRealisation);
+            else
+                throw Error(
+                    "Unrealised derivation '%s'", currentDep.to_string());
+        }
+        return res;
+    };
+
+    computeClosure<Realisation>(
+        startOutputs, res,
+        [&](const Realisation& current,
+            std::function<void(std::promise<std::set<Realisation>>&)>
+                processEdges) {
+            std::promise<std::set<Realisation>> promise;
+            try {
+                auto res = getDeps(current);
+                promise.set_value(res);
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+            return processEdges(promise);
+        });
+}
+
 nlohmann::json Realisation::toJSON() const {
+    auto jsonDependentRealisations = nlohmann::json::object();
+    for (auto & [depId, depOutPath] : dependentRealisations)
+        jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
     return nlohmann::json{
         {"id", id.to_string()},
         {"outPath", outPath.to_string()},
         {"signatures", signatures},
+        {"dependentRealisations", jsonDependentRealisations},
     };
 }
 
@@ -51,10 +93,16 @@ Realisation Realisation::fromJSON(
     if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
         signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
 
+    std::map <DrvOutput, StorePath> dependentRealisations;
+    if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end())
+        for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>())
+            dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
+
     return Realisation{
         .id = DrvOutput::parse(getField("id")),
         .outPath = StorePath(getField("outPath")),
         .signatures = signatures,
+        .dependentRealisations = dependentRealisations,
     };
 }
 
@@ -92,6 +140,24 @@ StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }
 
+bool Realisation::isCompatibleWith(const Realisation & other) const
+{
+    assert (id == other.id);
+    if (outPath == other.outPath) {
+        if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
+            warn(
+                "Encountered a realisation for '%s' with an empty set of "
+                "dependencies. This is likely an artifact from an older Nix. "
+                "I’ll try to fix the realisation if I can",
+                id.to_string());
+            return true;
+        } else if (dependentRealisations == other.dependentRealisations) {
+            return true;
+        }
+    }
+    return false;
+}
+
 void RealisedPath::closure(
     Store& store,
     const RealisedPath::Set& startPaths,

src/libstore/realisation.hh

@@ -28,6 +28,14 @@ struct Realisation {
 
     StringSet signatures;
 
+    /**
+     * The realisations that are required for the current one to be valid.
+     *
+     * When importing this realisation, the store will first check that all its
+     * dependencies exist, and map to the correct output path
+     */
+    std::map<DrvOutput, StorePath> dependentRealisations;
+
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
@@ -36,6 +44,11 @@ struct Realisation {
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 
+    static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
+    static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
+
+    bool isCompatibleWith(const Realisation & other) const;
+
     StorePath getPath() const { return outPath; }
 
     GENERATE_CMP(Realisation, me->id, me->outPath);

src/libstore/remote-store.cc

@@ -653,8 +653,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
     conn->to << wopRegisterDrvOutput;
-    conn->to << info.id.to_string();
-    conn->to << std::string(info.outPath.to_string());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        conn->to << info.id.to_string();
+        conn->to << std::string(info.outPath.to_string());
+    } else {
+        worker_proto::write(*this, conn->to, info);
+    }
     conn.processStderr();
 }
 
@@ -664,10 +668,17 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();
-    auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
-    if (outPaths.empty())
-        return std::nullopt;
-    return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
+        if (outPaths.empty())
+            return std::nullopt;
+        return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+    } else {
+        auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
+        if (realisations.empty())
+            return std::nullopt;
+        return *realisations.begin();
+    }
 }
 
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)

src/libstore/store-api.cc

@@ -337,6 +337,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     return info;
 }
 
+StringSet StoreConfig::getDefaultSystemFeatures()
+{
+    auto res = settings.systemFeatures.get();
+    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+        res.insert("ca-derivations");
+    return res;
+}
 
 Store::Store(const Params & params)
     : StoreConfig(params)
@@ -627,6 +634,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
 }
 
 
+StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths)
+{
+    StorePathSet paths;
+
+    for (auto & storePath : storePaths) {
+        if (!inputPaths.count(storePath))
+            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath));
+
+        computeFSClosure({storePath}, paths);
+    }
+
+    /* If there are derivations in the graph, then include their
+       outputs as well.  This is useful if you want to do things
+       like passing all build-time dependencies of some path to a
+       derivation that builds a NixOS DVD image. */
+    auto paths2 = paths;
+
+    for (auto & j : paths2) {
+        if (j.isDerivation()) {
+            Derivation drv = derivationFromPath(j);
+            for (auto & k : drv.outputsAndOptPaths(*this)) {
+                if (!k.second.second)
+                    /* FIXME: I am confused why we are calling
+                       `computeFSClosure` on the output path, rather than
+                       derivation itself. That doesn't seem right to me, so I
+                       won't try to implemented this for CA derivations. */
+                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
+                computeFSClosure(*k.second.second, paths);
+            }
+        }
+    }
+
+    return paths;
+}
+
+
 void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
     bool includeImpureInfo, bool showClosureSize,
     Base hashBase,
@@ -780,20 +823,39 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
     StorePathSet storePaths;
-    std::set<Realisation> realisations;
+    std::set<Realisation> toplevelRealisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto realisation = std::get_if<Realisation>(&path.raw)) {
             settings.requireExperimentalFeature("ca-derivations");
-            realisations.insert(*realisation);
+            toplevelRealisations.insert(*realisation);
         }
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+
+    ThreadPool pool;
+
     try {
-        for (auto & realisation : realisations) {
-            dstStore->registerDrvOutput(realisation, checkSigs);
-        }
-    } catch (MissingExperimentalFeature & e) {
+        // Copy the realisation closure
+        processGraph<Realisation>(
+            pool, Realisation::closure(*srcStore, toplevelRealisations),
+            [&](const Realisation & current) -> std::set<Realisation> {
+                std::set<Realisation> children;
+                for (const auto & [drvOutput, _] : current.dependentRealisations) {
+                    auto currentChild = srcStore->queryRealisation(drvOutput);
+                    if (!currentChild)
+                        throw Error(
+                            "incomplete realisation closure: '%s' is a "
+                            "dependency of '%s' but isn't registered",
+                            drvOutput.to_string(), current.id.to_string());
+                    children.insert(*currentChild);
+                }
+                return children;
+            },
+            [&](const Realisation& current) -> void {
+                dstStore->registerDrvOutput(current, checkSigs);
+            });
+    } catch (MissingExperimentalFeature& e) {
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
         // to at least copy the output paths.

src/libstore/store-api.hh

@@ -180,6 +180,8 @@ struct StoreConfig : public Config
 
     StoreConfig() = delete;
 
+    StringSet getDefaultSystemFeatures();
+
     virtual ~StoreConfig() { }
 
     virtual const std::string name() = 0;
@@ -196,7 +198,7 @@ struct StoreConfig : public Config
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
 
-    Setting<StringSet> systemFeatures{this, settings.systemFeatures,
+    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
         "system-features",
         "Optional features that the system this store builds on implements (like \"kvm\")."};
 
@@ -541,7 +543,7 @@ public:
     /* Add a store path as a temporary root of the garbage collector.
        The root disappears as soon as we exit. */
     virtual void addTempRoot(const StorePath & path)
-    { warn("not creating temp root, store doesn't support GC"); }
+    { debug("not creating temporary root, store doesn't support GC"); }
 
     /* Add an indirect root, which is merely a symlink to `path' from
        /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed
@@ -695,6 +697,11 @@ public:
 
     const Stats & getStats();
 
+    /* Computes the full closure of of a set of store-paths for e.g.
+       derivations that need this information for `exportReferencesGraph`.
+     */
+    StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
+
     /* Return the build log of the specified store path, if available,
        or null otherwise. */
     virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
@@ -760,7 +767,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
 std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
-    const StorePathSet& paths,
+    const StorePathSet & paths,
     RepairFlag repair = NoRepair,
     CheckSigsFlag checkSigs = CheckSigs,
     SubstituteFlag substitute = NoSubstitute);
@@ -864,4 +871,9 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
 
 std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
 
+std::map<DrvOutput, StorePath> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath);
+
 }

src/libstore/worker-protocol.hh

@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 30)
+#define PROTOCOL_VERSION (1 << 8 | 31)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

src/libutil/comparator.hh

@@ -25,6 +25,8 @@
     }
 #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
 #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
     GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args)
+    GENERATE_LEQ(args) \
+    GENERATE_NEQ(args)

src/libutil/config.cc

@@ -152,6 +152,16 @@ nlohmann::json Config::toJSON()
     return res;
 }
 
+std::string Config::toKeyValue()
+{
+    auto res = std::string();
+    for (auto & s : _settings)
+        if (!s.second.isAlias) {
+            res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
+        }
+    return res;
+}
+
 void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings)
@@ -385,6 +395,16 @@ nlohmann::json GlobalConfig::toJSON()
     return res;
 }
 
+std::string GlobalConfig::toKeyValue()
+{
+    std::string res;
+    std::map<std::string, Config::SettingInfo> settings;
+    globalConfig.getSettings(settings);
+    for (auto & s : settings)
+        res += fmt("%s = %s\n", s.first, s.second.value);
+    return res;
+}
+
 void GlobalConfig::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & config : *configRegistrations)

src/libutil/config.hh

@@ -99,6 +99,12 @@ public:
      */
     virtual nlohmann::json toJSON() = 0;
 
+    /**
+     * Outputs all settings in a key-value pair format suitable to be used as
+     * `nix.conf`
+     */
+    virtual std::string toKeyValue() = 0;
+
     /**
      * Converts settings to `Args` to be used on the command line interface
      * - args: args to write to
@@ -169,6 +175,8 @@ public:
 
     nlohmann::json toJSON() override;
 
+    std::string toKeyValue() override;
+
     void convertToArgs(Args & args, const std::string & category) override;
 };
 
@@ -330,6 +338,8 @@ struct GlobalConfig : public AbstractConfig
 
     nlohmann::json toJSON() override;
 
+    std::string toKeyValue() override;
+
     void convertToArgs(Args & args, const std::string & category) override;
 
     struct Register

src/libutil/logging.cc

@@ -46,7 +46,7 @@ public:
         : printBuildLogs(printBuildLogs)
     {
         systemd = getEnv("IN_SYSTEMD") == "1";
-        tty = isatty(STDERR_FILENO);
+        tty = shouldANSI();
     }
 
     bool isVerbose() override {

src/libutil/ref.hh

@@ -73,6 +73,16 @@ public:
         return ref<T2>((std::shared_ptr<T2>) p);
     }
 
+    bool operator == (const ref<T> & other) const
+    {
+        return p == other.p;
+    }
+
+    bool operator != (const ref<T> & other) const
+    {
+        return p != other.p;
+    }
+
 private:
 
     template<typename T2, typename... Args>

src/libutil/url.cc

@@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url)
         auto isFile = scheme.find("file") != std::string::npos;
 
         if (authority && *authority != "" && isFile)
-            throw Error("file:// URL '%s' has unexpected authority '%s'",
+            throw BadURL("file:// URL '%s' has unexpected authority '%s'",
                 url, *authority);
 
         if (isFile && path.empty())

src/libutil/util.cc

@@ -413,7 +413,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
         }
 
         int fd = openat(parentfd, path.c_str(), O_RDONLY);
-        if (!fd)
+        if (fd == -1)
             throw SysError("opening directory '%1%'", path);
         AutoCloseDir dir(fdopendir(fd));
         if (!dir)
@@ -435,12 +435,9 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed)
     if (dir == "")
         dir = "/";
 
-    AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY));
+    AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
     if (!dirfd) {
-        // This really shouldn't fail silently, but it's left this way
-        // for backwards compatibility.
         if (errno == ENOENT) return;
-
         throw SysError("opening directory '%1%'", path);
     }
 
@@ -1372,6 +1369,12 @@ void ignoreException()
     }
 }
 
+bool shouldANSI()
+{
+    return isatty(STDERR_FILENO)
+        && getEnv("TERM").value_or("dumb") != "dumb"
+        && !getEnv("NO_COLOR").has_value();
+}
 
 std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
 {

src/libutil/util.hh

@@ -482,6 +482,9 @@ constexpr char treeLast[] = "└───";
 constexpr char treeLine[] = "│   ";
 constexpr char treeNull[] = "    ";
 
+/* Determine whether ANSI escape sequences are appropriate for the
+   present output. */
+bool shouldANSI();
 
 /* Truncate a string to 'width' printable characters. If 'filterAll'
    is true, all ANSI escape sequences are filtered out. Otherwise,

src/nix-build/nix-build.cc

@@ -1,10 +1,15 @@
 #include <cstring>
 #include <fstream>
 #include <iostream>
+#include <filesystem>
 #include <regex>
 #include <sstream>
 #include <vector>
+#include <map>
 
+#include <nlohmann/json.hpp>
+
+#include "parsed-derivations.hh"
 #include "store-api.hh"
 #include "local-fs-store.hh"
 #include "globals.hh"
@@ -387,6 +392,12 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            auto resolvedDrv = drv.tryResolve(*store);
+            assert(resolvedDrv && "Successfully resolved the derivation");
+            drv = *resolvedDrv;
+        }
+
         // Set the environment.
         auto env = getEnv();
 
@@ -422,12 +433,45 @@ static void main_nix_build(int argc, char * * argv)
             } else
                 env[var.first] = var.second;
 
+        std::string structuredAttrsRC;
+
+        if (env.count("__json")) {
+            StorePathSet inputs;
+            for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) {
+                auto outputs = store->queryPartialDerivationOutputMap(depDrvPath);
+                for (auto & i : wantedDepOutputs) {
+                    auto o = outputs.at(i);
+                    store->computeFSClosure(*o, inputs);
+                }
+            }
+
+            ParsedDerivation parsedDrv(
+                StorePath(store->parseStorePath(drvInfo.queryDrvPath())),
+                drv
+            );
+
+            if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) {
+                auto json = structAttrs.value();
+                structuredAttrsRC = writeStructuredAttrsShell(json);
+
+                auto attrsJSON = (Path) tmpDir + "/.attrs.json";
+                writeFile(attrsJSON, json.dump());
+
+                auto attrsSH = (Path) tmpDir + "/.attrs.sh";
+                writeFile(attrsSH, structuredAttrsRC);
+
+                env["NIX_ATTRS_SH_FILE"] = attrsSH;
+                env["NIX_ATTRS_JSON_FILE"] = attrsJSON;
+                keepTmp = true;
+            }
+        }
+
         /* Run a shell using the derivation's environment.  For
            convenience, source $stdenv/setup to setup additional
            environment variables and shell functions.  Also don't
            lose the current $PATH directories. */
         auto rcfile = (Path) tmpDir + "/rc";
-        writeFile(rcfile, fmt(
+        std::string rc = fmt(
                 R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s +
                 (keepTmp ?
                     "trap _nix_shell_clean_tmpdir EXIT; "
@@ -436,8 +480,9 @@ static void main_nix_build(int argc, char * * argv)
                     "_nix_shell_clean_tmpdir; ") +
                 (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") +
                 "%2%"
-                "dontAddDisableDepTrack=1; "
-                "[ -e $stdenv/setup ] && source $stdenv/setup; "
+                "dontAddDisableDepTrack=1;\n"
+                + structuredAttrsRC +
+                "\n[ -e $stdenv/setup ] && source $stdenv/setup; "
                 "%3%"
                 "PATH=%4%:\"$PATH\"; "
                 "SHELL=%5%; "
@@ -455,7 +500,9 @@ static void main_nix_build(int argc, char * * argv)
                 shellEscape(dirOf(*shell)),
                 shellEscape(*shell),
                 (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""),
-                envCommand));
+                envCommand);
+        vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc);
+        writeFile(rcfile, rc);
 
         Strings envStrs;
         for (auto & i : env)

src/nix/build.cc

@@ -69,8 +69,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                             store2->addPermRoot(bo.path, absPath(symlink));
                         },
                         [&](BuiltPath::Built bfd) {
-                            auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
-                            for (auto & output : builtOutputs) {
+                            for (auto & output : bfd.outputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);

src/nix/develop.cc

@@ -8,7 +8,7 @@
 #include "affinity.hh"
 #include "progress-bar.hh"
 
-#include <regex>
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -25,94 +25,142 @@ static DevelopSettings developSettings;
 
 static GlobalConfig::Register rDevelopSettings(&developSettings);
 
-struct Var
-{
-    bool exported = true;
-    bool associative = false;
-    std::string quoted; // quoted string or array
-};
-
 struct BuildEnvironment
 {
-    std::map<std::string, Var> env;
-    std::string bashFunctions;
-};
+    struct String
+    {
+        bool exported;
+        std::string value;
 
-BuildEnvironment readEnvironment(const Path & path)
-{
-    BuildEnvironment res;
+        bool operator == (const String & other) const
+        {
+            return exported == other.exported && value == other.value;
+        }
+    };
 
-    std::set<std::string> exported;
+    using Array = std::vector<std::string>;
 
-    debug("reading environment file '%s'", path);
+    using Associative = std::map<std::string, std::string>;
 
-    auto file = readFile(path);
+    using Value = std::variant<String, Array, Associative>;
 
-    auto pos = file.cbegin();
+    std::map<std::string, Value> vars;
+    std::map<std::string, std::string> bashFunctions;
 
-    static std::string varNameRegex =
-        R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
+    static BuildEnvironment fromJSON(std::string_view in)
+    {
+        BuildEnvironment res;
 
-    static std::string simpleStringRegex =
-        R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
+        std::set<std::string> exported;
 
-    static std::string dquotedStringRegex =
-        R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
+        auto json = nlohmann::json::parse(in);
 
-    static std::string squotedStringRegex =
-        R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
-
-    static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
-
-    static std::regex declareRegex(
-        "^declare -a?x (" + varNameRegex + ")(=(" +
-        dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
-
-    static std::regex varRegex(
-        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
-
-    /* Note: we distinguish between an indexed and associative array
-       using the space before the closing parenthesis. Will
-       undoubtedly regret this some day. */
-    static std::regex assocArrayRegex(
-        "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
-
-    static std::regex functionRegex(
-        "^" + varNameRegex + " \\(\\) *\n");
-
-    while (pos != file.end()) {
-
-        std::smatch match;
-
-        if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            exported.insert(match[1]);
+        for (auto & [name, info] : json["variables"].items()) {
+            std::string type = info["type"];
+            if (type == "var" || type == "exported")
+                res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }});
+            else if (type == "array")
+                res.vars.insert({name, (Array) info["value"]});
+            else if (type == "associative")
+                res.vars.insert({name, (Associative) info["value"]});
         }
 
-        else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
+        for (auto & [name, def] : json["bashFunctions"].items()) {
+            res.bashFunctions.insert({name, def});
         }
 
-        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
-        }
-
-        else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
-            res.bashFunctions = std::string(pos, file.cend());
-            break;
-        }
-
-        else throw Error("shell environment '%s' has unexpected line '%s'",
-            path, file.substr(pos - file.cbegin(), 60));
+        return res;
     }
 
-    res.env.erase("__output");
+    std::string toJSON() const
+    {
+        auto res = nlohmann::json::object();
 
-    return res;
-}
+        auto vars2 = nlohmann::json::object();
+        for (auto & [name, value] : vars) {
+            auto info = nlohmann::json::object();
+            if (auto str = std::get_if<String>(&value)) {
+                info["type"] = str->exported ? "exported" : "var";
+                info["value"] = str->value;
+            }
+            else if (auto arr = std::get_if<Array>(&value)) {
+                info["type"] = "array";
+                info["value"] = *arr;
+            }
+            else if (auto arr = std::get_if<Associative>(&value)) {
+                info["type"] = "associative";
+                info["value"] = *arr;
+            }
+            vars2[name] = std::move(info);
+        }
+        res["variables"] = std::move(vars2);
+
+        res["bashFunctions"] = bashFunctions;
+
+        auto json = res.dump();
+
+        assert(BuildEnvironment::fromJSON(json) == *this);
+
+        return json;
+    }
+
+    void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
+    {
+        for (auto & [name, value] : vars) {
+            if (!ignoreVars.count(name)) {
+                if (auto str = std::get_if<String>(&value)) {
+                    out << fmt("%s=%s\n", name, shellEscape(str->value));
+                    if (str->exported)
+                        out << fmt("export %s\n", name);
+                }
+                else if (auto arr = std::get_if<Array>(&value)) {
+                    out << "declare -a " << name << "=(";
+                    for (auto & s : *arr)
+                        out << shellEscape(s) << " ";
+                    out << ")\n";
+                }
+                else if (auto arr = std::get_if<Associative>(&value)) {
+                    out << "declare -A " << name << "=(";
+                    for (auto & [n, v] : *arr)
+                        out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " ";
+                    out << ")\n";
+                }
+            }
+        }
+
+        for (auto & [name, def] : bashFunctions) {
+            out << name << " ()\n{\n" << def << "}\n";
+        }
+    }
+
+    static std::string getString(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return str->value;
+        else
+            throw Error("bash variable is not a string");
+    }
+
+    static Array getStrings(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return tokenizeString<Array>(str->value);
+        else if (auto arr = std::get_if<Array>(&value)) {
+            return *arr;
+        } else if (auto assoc = std::get_if<Associative>(&value)) {
+            Array assocKeys;
+            std::for_each(assoc->begin(), assoc->end(), [&](auto & n) { assocKeys.push_back(n.first); });
+            return assocKeys;
+        }
+        else
+            throw Error("bash variable is not a string or array");
+    }
+
+    bool operator == (const BuildEnvironment & other) const
+    {
+        return vars == other.vars && bashFunctions == other.bashFunctions;
+    }
+};
 
 const static std::string getEnvSh =
     #include "get-env.sh.gen.hh"
@@ -144,17 +192,26 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs) {
-        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
-        drv.env[output.first] = "";
-    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        for (auto & output : drv.outputs) {
+            output.second = {
+                .output = DerivationOutputDeferred{},
+            };
+            drv.env[output.first] = hashPlaceholder(output.first);
+        }
+    } else {
+        for (auto & output : drv.outputs) {
+            output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+            drv.env[output.first] = "";
+        }
+        Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
 
-    for (auto & output : drv.outputs) {
-        auto outPath = store->makeOutputPath(output.first, h, drv.name);
-        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
-        drv.env[output.first] = store->printStorePath(outPath);
+        for (auto & output : drv.outputs) {
+            auto outPath = store->makeOutputPath(output.first, h, drv.name);
+            output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+            drv.env[output.first] = store->printStorePath(outPath);
+        }
     }
 
     auto shellDrvPath = writeDerivation(*store, drv);
@@ -162,8 +219,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Build the derivation. */
     store->buildPaths({DerivedPath::Built{shellDrvPath}});
 
-    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
-        auto & [_1, optPath] = outputAndOptPath;
+    for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) {
         assert(optPath);
         auto & outPath = *optPath;
         assert(store->isValidPath(outPath));
@@ -177,18 +233,15 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
 struct Common : InstallableCommand, MixProfile
 {
-    std::set<string> ignoreVars{
+    std::set<std::string> ignoreVars{
         "BASHOPTS",
-        "EUID",
         "HOME", // FIXME: don't ignore in pure mode?
-        "HOSTNAME",
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",
+        "NIX_REMOTE",
         "PPID",
-        "PWD",
         "SHELLOPTS",
-        "SHLVL",
         "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
         "TEMP",
         "TEMPDIR",
@@ -224,22 +277,10 @@ struct Common : InstallableCommand, MixProfile
 
         out << "nix_saved_PATH=\"$PATH\"\n";
 
-        for (auto & i : buildEnvironment.env) {
-            if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
-                if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
-                else {
-                    out << fmt("%s=%s\n", i.first, i.second.quoted);
-                    if (i.second.exported)
-                        out << fmt("export %s\n", i.first);
-                }
-            }
-        }
+        buildEnvironment.toBash(out, ignoreVars);
 
         out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
 
-        out << buildEnvironment.bashFunctions << "\n";
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
@@ -249,16 +290,16 @@ struct Common : InstallableCommand, MixProfile
         auto script = out.str();
 
         /* Substitute occurrences of output paths. */
-        auto outputs = buildEnvironment.env.find("outputs");
-        assert(outputs != buildEnvironment.env.end());
+        auto outputs = buildEnvironment.vars.find("outputs");
+        assert(outputs != buildEnvironment.vars.end());
 
         // FIXME: properly unquote 'outputs'.
         StringMap rewrites;
-        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
-            auto from = buildEnvironment.env.find(outputName);
-            assert(from != buildEnvironment.env.end());
+        for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) {
+            auto from = buildEnvironment.vars.find(outputName);
+            assert(from != buildEnvironment.vars.end());
             // FIXME: unquote
-            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+            rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName});
         }
 
         /* Substitute redirects. */
@@ -285,6 +326,12 @@ struct Common : InstallableCommand, MixProfile
     {
         return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()};
     }
+    Strings getDefaultFlakeAttrPathPrefixes() override
+    {
+        auto res = SourceExprCommand::getDefaultFlakeAttrPathPrefixes();
+        res.emplace_front("devShells." + settings.thisSystem.get());
+        return res;
+    }
 
     StorePath getShellOutPath(ref<Store> store)
     {
@@ -312,7 +359,9 @@ struct Common : InstallableCommand, MixProfile
 
         updateProfile(shellOutPath);
 
-        return {readEnvironment(strPath), strPath};
+        debug("reading environment file '%s'", strPath);
+
+        return {BuildEnvironment::fromJSON(readFile(strPath)), strPath};
     }
 };
 
@@ -434,13 +483,17 @@ struct CmdDevelop : Common, MixEnvironment
         try {
             auto state = getEvalState();
 
+            auto nixpkgsLockFlags = lockFlags;
+            nixpkgsLockFlags.inputOverrides = {};
+            nixpkgsLockFlags.inputUpdates = {};
+
             auto bashInstallable = std::make_shared<InstallableFlake>(
                 this,
                 state,
                 installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
-                lockFlags);
+                nixpkgsLockFlags);
 
             shell = state->store->printStorePath(
                 toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";
@@ -461,7 +514,7 @@ struct CmdDevelop : Common, MixEnvironment
     }
 };
 
-struct CmdPrintDevEnv : Common
+struct CmdPrintDevEnv : Common, MixJSON
 {
     std::string description() override
     {
@@ -483,7 +536,10 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        std::cout << makeRcScript(store, buildEnvironment);
+        logger->writeToStdout(
+            json
+            ? buildEnvironment.toJSON()
+            : makeRcScript(store, buildEnvironment));
     }
 };
 

src/nix/develop.md

@@ -84,11 +84,20 @@ the flake's `nixConfig` attribute.
 
 # Flake output attributes
 
-If no flake output attribute is given, `nix run` tries the following
+If no flake output attribute is given, `nix develop` tries the following
 flake output attributes:
 
 * `devShell.<system>`
 
 * `defaultPackage.<system>`
 
+If a flake output *name* is given, `nix develop` tries the following flake
+output attributes:
+
+* `devShells.<system>.<name>`
+
+* `packages.<system>.<name>`
+
+* `legacyPackages.<system>.<name>`
+
 )""

src/nix/edit.cc

@@ -31,7 +31,7 @@ struct CmdEdit : InstallableCommand
         auto [v, pos] = installable->toValue(*state);
 
         try {
-            pos = findDerivationFilename(*state, *v, installable->what());
+            pos = findPackageFilename(*state, *v, installable->what());
         } catch (NoPositionInfo &) {
         }
 

src/nix/flake-check.md

@@ -33,6 +33,7 @@ The following flake output attributes must be derivations:
 * `checks.`*system*`.`*name*
 * `defaultPackage.`*system*`
 * `devShell.`*system*`
+* `devShells.`*system*`.`*name*`
 * `nixosConfigurations.`*name*`.config.system.build.toplevel
 * `packages.`*system*`.`*name*
 

src/nix/flake.cc

@@ -84,6 +84,7 @@ struct CmdFlakeUpdate : FlakeCommand
 
         lockFlags.recreateLockFile = true;
         lockFlags.writeLockFile = true;
+        lockFlags.applyNixConfig = true;
 
         lockFlake();
     }
@@ -114,6 +115,7 @@ struct CmdFlakeLock : FlakeCommand
         settings.tarballTtl = 0;
 
         lockFlags.writeLockFile = true;
+        lockFlags.applyNixConfig = true;
 
         lockFlake();
     }
@@ -270,6 +272,8 @@ struct CmdFlakeCheck : FlakeCommand
         settings.readOnlyMode = !build;
 
         auto state = getEvalState();
+
+        lockFlags.applyNixConfig = true;
         auto flake = lockFlake();
 
         bool hasErrors = false;
@@ -482,7 +486,7 @@ struct CmdFlakeCheck : FlakeCommand
                             }
                         }
 
-                        else if (name == "packages") {
+                        else if (name == "packages" || name == "devShells") {
                             state->forceAttrs(vOutput, pos);
                             for (auto & attr : *vOutput.attrs) {
                                 checkSystemName(attr.name, *attr.pos);
@@ -910,6 +914,7 @@ struct CmdFlakeShow : FlakeCommand
                     logger->cout("%s: %s '%s'",
                         headerPrefix,
                         attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" :
+                        attrPath.size() >= 2 && attrPath[0] == "devShells" ? "development environment" :
                         attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" :
                         attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" :
                         "package",
@@ -928,6 +933,7 @@ struct CmdFlakeShow : FlakeCommand
                     || ((attrPath.size() == 1 || attrPath.size() == 2)
                         && (attrPath[0] == "checks"
                             || attrPath[0] == "packages"
+                            || attrPath[0] == "devShells"
                             || attrPath[0] == "apps"))
                     )
                 {
@@ -936,7 +942,7 @@ struct CmdFlakeShow : FlakeCommand
 
                 else if (
                     (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell"))
-                    || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages"))
+                    || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells"))
                     )
                 {
                     if (visitor.isDerivation())

src/nix/get-env.sh

@@ -8,12 +8,123 @@ if [[ -n $stdenv ]]; then
     source $stdenv/setup
 fi
 
-for __output in $outputs; do
+# Better to use compgen, but stdenv bash doesn't have it.
+__vars="$(declare -p)"
+__functions="$(declare -F)"
+
+__dumpEnv() {
+    printf '{\n'
+
+    printf '  "bashFunctions": {\n'
+    local __first=1
+    while read __line; do
+        if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi
+        __fun_name="${BASH_REMATCH[1]}"
+        __fun_body="$(type $__fun_name)"
+        if [[ $__fun_body =~ \{(.*)\} ]]; then
+            if [[ -z $__first ]]; then printf ',\n'; else __first=; fi
+            __fun_body="${BASH_REMATCH[1]}"
+            printf "    "
+            __escapeString "$__fun_name"
+            printf ':'
+            __escapeString "$__fun_body"
+        else
+            printf "Cannot parse definition of function '%s'.\n" "$__fun_name" >&2
+            return 1
+        fi
+    done < <(printf "%s\n" "$__functions")
+    printf '\n  },\n'
+
+    printf '  "variables": {\n'
+    local __first=1
+    while read __line; do
+        if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi
+        local type="${BASH_REMATCH[1]}"
+        local __var_name="${BASH_REMATCH[2]}"
+
+        if [[ $__var_name =~ ^BASH_ || \
+              $__var_name = _ || \
+              $__var_name = DIRSTACK || \
+              $__var_name = EUID || \
+              $__var_name = FUNCNAME || \
+              $__var_name = HISTCMD || \
+              $__var_name = HOSTNAME || \
+              $__var_name = GROUPS || \
+              $__var_name = PIPESTATUS || \
+              $__var_name = PWD || \
+              $__var_name = RANDOM || \
+              $__var_name = SHLVL || \
+              $__var_name = SECONDS \
+            ]]; then continue; fi
+
+        if [[ -z $__first ]]; then printf ',\n'; else __first=; fi
+
+        printf "    "
+        __escapeString "$__var_name"
+        printf ': {'
+
+        # FIXME: handle -i, -r, -n.
+        if [[ $type == -x ]]; then
+            printf '"type": "exported", "value": '
+            __escapeString "${!__var_name}"
+        elif [[ $type == -- ]]; then
+            printf '"type": "var", "value": '
+            __escapeString "${!__var_name}"
+        elif [[ $type == -a ]]; then
+            printf '"type": "array", "value": ['
+            local __first2=1
+            __var_name="$__var_name[@]"
+            for __i in "${!__var_name}"; do
+                if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi
+                __escapeString "$__i"
+                printf ' '
+            done
+            printf ']'
+        elif [[ $type == -A ]]; then
+            printf '"type": "associative", "value": {\n'
+            local __first2=1
+            declare -n __var_name2="$__var_name"
+            for __i in "${!__var_name2[@]}"; do
+                if [[ -z $__first2 ]]; then printf ',\n'; else __first2=; fi
+                printf "      "
+                __escapeString "$__i"
+                printf ": "
+                __escapeString "${__var_name2[$__i]}"
+            done
+            printf '\n    }'
+        else
+            printf '"type": "unknown"'
+        fi
+
+        printf "}"
+    done < <(printf "%s\n" "$__vars")
+    printf '\n  }\n}'
+}
+
+__escapeString() {
+    local __s="$1"
+    __s="${__s//\\/\\\\}"
+    __s="${__s//\"/\\\"}"
+    __s="${__s//$'\n'/\\n}"
+    __s="${__s//$'\r'/\\r}"
+    __s="${__s//$'\t'/\\t}"
+    printf '"%s"' "$__s"
+}
+
+# In case of `__structuredAttrs = true;` the list of outputs is an associative
+# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist`
+# must contain the array's keys (hence `${!...[@]}`) in this case.
+if [ -e .attrs.sh ]; then
+    __olist="${!outputs[@]}"
+else
+    __olist=$outputs
+fi
+
+for __output in $__olist; do
     if [[ -z $__done ]]; then
-        export > ${!__output}
-        set >> ${!__output}
+        __dumpEnv > ${!__output}
         __done=1
     else
-        echo -n >> ${!__output}
+        echo -n >> "${!__output}"
     fi
 done

src/nix/prefetch.cc

@@ -283,8 +283,6 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
         expectArg("url", &url);
     }
 
-    Category category() override { return catUtility; }
-
     std::string description() override
     {
         return "download a file into the Nix store";

src/nix/print-dev-env.md

@@ -8,12 +8,43 @@ R""(
   # . <(nix print-dev-env nixpkgs#hello)
   ```
 
+* Get the build environment in JSON format:
+
+  ```console
+  # nix print-dev-env nixpkgs#hello --json
+  ```
+
+  The output will look like this:
+
+  ```json
+  {
+    "bashFunctions": {
+      "buildPhase": " \n    runHook preBuild;\n...",
+      ...
+    },
+    "variables": {
+      "src": {
+        "type": "exported",
+        "value": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz"
+      },
+      "postUnpackHooks": {
+        "type": "array",
+        "value": ["_updateSourceDateEpochFromSourceRoot"]
+      },
+      ...
+    }
+  }
+  ```
+
 # Description
 
-This command prints a shell script that can be sourced by `b`ash and
-that sets the environment variables and shell functions defined by the
-build process of *installable*. This allows you to get a similar build
+This command prints a shell script that can be sourced by `bash` and
+that sets the variables and shell functions defined by the build
+process of *installable*. This allows you to get a similar build
 environment in your current shell rather than in a subshell (as with
 `nix develop`).
 
+With `--json`, the output is a JSON serialisation of the variables and
+functions defined by the build process.
+
 )""

src/nix/profile-remove.md

@@ -15,6 +15,7 @@ R""(
   ```
 
 * Remove all packages:
+
   ```console
   # nix profile remove '.*'
   ```

src/nix/registry-add.md

@@ -21,6 +21,13 @@ R""(
   # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs
   ```
 
+* Add `nixpkgs` pointing to `github:nixos/nixpkgs` to your custom flake
+  registry:
+
+  ```console
+  nix registry add --registry ./custom-flake-registry.json nixpkgs github:nixos/nixpkgs
+  ```
+
 # Description
 
 This command adds an entry to the user registry that maps flake

src/nix/registry-pin.md

@@ -24,6 +24,13 @@ R""(
   …
   ```
 
+* Pin `nixpkgs` in a custom registry to its most recent Git revision:
+
+  ```console
+  # nix registry pin --registry ./custom-flake-registry.json nixpkgs
+  ```
+
+
 # Description
 
 This command adds an entry to the user registry that maps flake

src/nix/registry-remove.md

@@ -8,6 +8,12 @@ R""(
   # nix registry remove nixpkgs
   ```
 
+* Remove the entry `nixpkgs` from a custom registry:
+
+  ```console
+  # nix registry remove --registry ./custom-flake-registry.json nixpkgs
+  ```
+
 # Description
 
 This command removes from the user registry any entry for flake

src/nix/registry.cc

@@ -10,6 +10,46 @@
 using namespace nix;
 using namespace nix::flake;
 
+
+class RegistryCommand : virtual Args
+{
+    std::string registry_path;
+
+    std::shared_ptr<fetchers::Registry> registry;
+
+public:
+
+    RegistryCommand()
+    {
+        addFlag({
+            .longName = "registry",
+            .description = "The registry to operate on.",
+            .labels = {"registry"},
+            .handler = {&registry_path},
+        });
+    }
+
+    std::shared_ptr<fetchers::Registry> getRegistry()
+    {
+        if (registry) return registry;
+        if (registry_path.empty()) {
+            registry = fetchers::getUserRegistry();
+        } else {
+            registry = fetchers::getCustomRegistry(registry_path);
+        }
+        return registry;
+    }
+
+    Path getRegistryPath()
+    {
+        if (registry_path.empty()) {
+            return fetchers::getUserRegistryPath();
+        } else {
+            return registry_path;
+        }
+    }
+};
+
 struct CmdRegistryList : StoreCommand
 {
     std::string description() override
@@ -45,7 +85,7 @@ struct CmdRegistryList : StoreCommand
     }
 };
 
-struct CmdRegistryAdd : MixEvalArgs, Command
+struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand
 {
     std::string fromUrl, toUrl;
 
@@ -71,16 +111,16 @@ struct CmdRegistryAdd : MixEvalArgs, Command
     {
         auto fromRef = parseFlakeRef(fromUrl);
         auto toRef = parseFlakeRef(toUrl);
+        auto registry = getRegistry();
         fetchers::Attrs extraAttrs;
         if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir;
-        auto userRegistry = fetchers::getUserRegistry();
-        userRegistry->remove(fromRef.input);
-        userRegistry->add(fromRef.input, toRef.input, extraAttrs);
-        userRegistry->write(fetchers::getUserRegistryPath());
+        registry->remove(fromRef.input);
+        registry->add(fromRef.input, toRef.input, extraAttrs);
+        registry->write(getRegistryPath());
     }
 };
 
-struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
+struct CmdRegistryRemove : RegistryCommand, Command
 {
     std::string url;
 
@@ -103,19 +143,21 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
 
     void run() override
     {
-        auto userRegistry = fetchers::getUserRegistry();
-        userRegistry->remove(parseFlakeRef(url).input);
-        userRegistry->write(fetchers::getUserRegistryPath());
+        auto registry = getRegistry();
+        registry->remove(parseFlakeRef(url).input);
+        registry->write(getRegistryPath());
     }
 };
 
-struct CmdRegistryPin : virtual Args, EvalCommand
+struct CmdRegistryPin : RegistryCommand, EvalCommand
 {
     std::string url;
 
+    std::string locked;
+
     std::string description() override
     {
-        return "pin a flake to its current version in user flake registry";
+        return "pin a flake to its current version or to the current version of a flake URL";
     }
 
     std::string doc() override
@@ -128,18 +170,31 @@ struct CmdRegistryPin : virtual Args, EvalCommand
     CmdRegistryPin()
     {
         expectArg("url", &url);
+
+        expectArgs({
+            .label = "locked",
+            .optional = true,
+            .handler = {&locked},
+            .completer = {[&](size_t, std::string_view prefix) {
+                completeFlakeRef(getStore(), prefix);
+            }}
+        });
     }
 
     void run(nix::ref<nix::Store> store) override
     {
+        if (locked.empty()) {
+            locked = url;
+        }
+        auto registry = getRegistry();
         auto ref = parseFlakeRef(url);
-        auto userRegistry = fetchers::getUserRegistry();
-        userRegistry->remove(ref.input);
-        auto [tree, resolved] = ref.resolve(store).input.fetch(store);
+        auto locked_ref = parseFlakeRef(locked);
+        registry->remove(ref.input);
+        auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store);
         fetchers::Attrs extraAttrs;
         if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
-        userRegistry->add(ref.input, resolved, extraAttrs);
-        userRegistry->write(fetchers::getUserRegistryPath());
+        registry->add(ref.input, resolved, extraAttrs);
+        registry->write(getRegistryPath());
     }
 };