Merge pull request #11586 from NixOS/mergify/bp/2.18-maintenance/pr-11585

builtin:fetchurl: Enable TLS verification (backport #11585)

.github/labeler.yml

@@ -20,4 +20,4 @@
   # Unit tests
   - src/*/tests/**/*
   # Functional and integration tests
-  - tests/**/*
+  - tests/functional/**/*

.gitignore

@@ -41,14 +41,14 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
-/src/libexpr/tests/libnixexpr-tests
+/tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
-/src/libstore/tests/libnixstore-tests
+/tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
-/src/libutil/tests/libnixutil-tests
+/tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
 
@@ -79,24 +79,24 @@ perl/Makefile.config
 
 /src/build-remote/build-remote
 
-# /tests/
-/tests/test-tmp
-/tests/common/vars-and-functions.sh
-/tests/result*
-/tests/restricted-innocent
-/tests/shell
-/tests/shell.drv
-/tests/config.nix
-/tests/ca/config.nix
-/tests/dyn-drv/config.nix
-/tests/repl-result-out
-/tests/test-libstoreconsumer/test-libstoreconsumer
+# /tests/functional/
+/tests/functional/test-tmp
+/tests/functional/common/vars-and-functions.sh
+/tests/functional/result*
+/tests/functional/restricted-innocent
+/tests/functional/shell
+/tests/functional/shell.drv
+/tests/functional/config.nix
+/tests/functional/ca/config.nix
+/tests/functional/dyn-drv/config.nix
+/tests/functional/repl-result-out
+/tests/functional/test-libstoreconsumer/test-libstoreconsumer
 
-# /tests/lang/
-/tests/lang/*.out
-/tests/lang/*.out.xml
-/tests/lang/*.err
-/tests/lang/*.ast
+# /tests/functional/lang/
+/tests/functional/lang/*.out
+/tests/functional/lang/*.out.xml
+/tests/functional/lang/*.err
+/tests/functional/lang/*.ast
 
 /perl/lib/Nix/Config.pm
 /perl/lib/Nix/Store.cc

.version

@@ -1 +1 @@
-2.18.0
+2.18.8

CONTRIBUTING.md

@@ -52,7 +52,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
 
    - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue
    - [ ] Tests, as appropriate:
-     - Functional tests – [`tests/**.sh`](./tests)
+     - Functional tests – [`tests/functional/**.sh`](./tests/functional)
      - Unit tests – [`src/*/tests`](./src/)
      - Integration tests – [`tests/nixos/*`](./tests/nixos)
    - [ ] User documentation in the [manual](..doc/manual/src)

Makefile

@@ -23,14 +23,17 @@ makefiles = \
 
 ifeq ($(tests), yes)
 makefiles += \
-  src/libutil/tests/local.mk \
-  src/libstore/tests/local.mk \
-  src/libexpr/tests/local.mk \
-  tests/local.mk \
-  tests/ca/local.mk \
-  tests/dyn-drv/local.mk \
-  tests/test-libstoreconsumer/local.mk \
-  tests/plugins/local.mk
+  tests/unit/libutil/local.mk \
+  tests/unit/libutil-support/local.mk \
+  tests/unit/libstore/local.mk \
+  tests/unit/libstore-support/local.mk \
+  tests/unit/libexpr/local.mk \
+  tests/unit/libexpr-support/local.mk \
+  tests/functional/local.mk \
+  tests/functional/ca/local.mk \
+  tests/functional/dyn-drv/local.mk \
+  tests/functional/test-libstoreconsumer/local.mk \
+  tests/functional/plugins/local.mk
 else
 makefiles += \
   mk/disable-tests.mk

configure.ac

@@ -252,6 +252,17 @@ case "$host_os" in
                         [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
       have_seccomp=1
       AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
+      AC_COMPILE_IFELSE([
+        AC_LANG_SOURCE([[
+          #include <seccomp.h>
+          #ifndef __SNR_fchmodat2
+          # error "Missing support for fchmodat2"
+          #endif
+        ]])
+      ], [], [
+        echo "libseccomp is missing __SNR_fchmodat2. Please provide libseccomp 2.5.5 or later"
+        exit 1
+      ])
     else
       have_seccomp=
     fi

doc/internal-api/doxygen.cfg.in

@@ -39,17 +39,21 @@ INPUT                  = \
   src/libcmd \
   src/libexpr \
   src/libexpr/flake \
-  src/libexpr/tests \
-  src/libexpr/tests/value \
+  tests/unit/libexpr \
+  tests/unit/libexpr/value \
+  tests/unit/libexpr/test \
+  tests/unit/libexpr/test/value \
   src/libexpr/value \
   src/libfetchers \
   src/libmain \
   src/libstore \
   src/libstore/build \
   src/libstore/builtins \
-  src/libstore/tests \
+  tests/unit/libstore \
+  tests/unit/libstore/test \
   src/libutil \
-  src/libutil/tests \
+  tests/unit/libutil \
+  tests/unit/libutil/test \
   src/nix \
   src/nix-env \
   src/nix-store

doc/manual/custom.css

@@ -1,3 +1,25 @@
+:root {
+    --sidebar-width: 23em;
+}
+
+h1.menu-title::before {
+  content: "";
+  background-image: url("./favicon.svg");
+  padding: 1.25em;
+  background-position: center center;
+  background-size: 2em;
+  background-repeat: no-repeat;
+}
+
+
+h1.menu-title {
+  padding: 0.5em;
+}
+
+.sidebar .sidebar-scrollbox {
+  padding: 1em;
+}
+
 h1:not(:first-of-type) {
     margin-top: 1.3em;
 }

doc/manual/generate-manpage.nix

@@ -5,7 +5,7 @@ let
   inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings;
 in
 
-commandDump:
+inlineHTML: commandDump:
 
 let
 
@@ -30,7 +30,7 @@ let
 
         ${maybeSubcommands}
 
-        ${maybeDocumentation}
+        ${maybeStoreDocs}
 
         ${maybeOptions}
       '';
@@ -70,7 +70,7 @@ let
         * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description}
       '';
 
-      maybeDocumentation = optionalString
+      maybeStoreDocs = optionalString
         (details ? doc)
         (replaceStrings ["@stores@"] [storeDocs] details.doc);
 
@@ -91,17 +91,20 @@ let
           listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts));
           showOption = name: option:
             let
+              result = trim ''
+                - ${item}
+                  ${option.description}
+              '';
+              item = if inlineHTML
+                then ''<span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}''
+                else "`--${name}` ${shortName} ${labels}";
               shortName = optionalString
                 (option ? shortName)
                 ("/ `-${option.shortName}`");
               labels = optionalString
                 (option ? labels)
                 (concatStringsSep " " (map (s: "*${s}*") option.labels));
-            in trim ''
-              - <span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}
-
-                ${option.description}
-            '';
+            in result;
           categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions)));
         in concatStrings (map showCategory categories);
     in squash result;
@@ -162,7 +165,7 @@ let
 
           **Settings**:
 
-          ${showSettings { useAnchors = false; } settings}
+          ${showSettings { inherit inlineHTML; } settings}
         '';
     in concatStrings (attrValues (mapAttrs showStore commandInfo.stores));
 

doc/manual/local.mk

@@ -98,12 +98,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/sr
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@ $@.tmp
-	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)'
+	$(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)'
 	@mv $@.tmp $@
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
-	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
+	$(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { inlineHTML = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
 	@mv $@.tmp $@
 
 $(d)/nix.json: $(bindir)/nix
@@ -173,7 +173,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 	done
 	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/favicon.png $(d)/src/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/redirects.js

@@ -1,7 +1,9 @@
-// redirect rules for anchors ensure backwards compatibility of URLs.
-// this must be done on the client side, as web servers do not see the anchor part of the URL.
+// redirect rules for URL fragments (client-side) to prevent link rot.
+// this must be done on the client side, as web servers do not see the fragment part of the URL.
+// it will only work with JavaScript enabled in the browser, but this is the best we can do here.
+// see ./_redirects for path redirects (client-side)
 
-// redirections are declared as follows:
+// redirects are declared as follows:
 // each entry has as its key a path matching the requested URL path, relative to the mdBook document root.
 //
 //     IMPORTANT: it must specify the full path with file name and suffix
@@ -19,6 +21,7 @@ const redirects = {
     "chap-distributed-builds": "advanced-topics/distributed-builds.html",
     "chap-post-build-hook": "advanced-topics/post-build-hook.html",
     "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats",
+    "chap-writing-nix-expressions": "language/index.html",
     "part-command-ref": "command-ref/command-ref.html",
     "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation",
     "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges",

doc/manual/rl-next/harden-user-sandboxing.md

@@ -0,0 +1,8 @@
+---
+synopsis: Harden the user sandboxing
+significance: significant
+issues:
+prs: <only provided once merged>
+---
+
+The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user.

doc/manual/rl-next/leading-period.md

@@ -0,0 +1,10 @@
+---
+synopsis: Store paths are allowed to start with `.`
+issues: 912
+prs: 9867 9091 9095 9120 9121 9122 9130 9219 9224
+---
+
+Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties.
+From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`.
+
+Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286).

doc/manual/rl-next/verify-tls.md

@@ -0,0 +1,8 @@
+---
+synopsis: "`<nix/fetchurl.nix>` uses TLS verification"
+prs: [11585]
+---
+
+Previously `<nix/fetchurl.nix>` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `<nix/fetchurl.nix>` will now fail if you're fetching from a HTTPS server that does not have a valid certificate.
+
+`<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.

doc/manual/src/SUMMARY.md.in

@@ -109,7 +109,6 @@
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md)
   - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
   - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)

doc/manual/src/_redirects

@@ -0,0 +1,30 @@
+# redirect rules for paths (server-side) to prevent link rot.
+# see ./redirects.js for redirects based on URL fragments (client-side)
+#
+# concrete user story this supports:
+# - user finds URL to the manual for Nix x.y
+# - Nix x.z (z > y) is the most recent release
+# - updating the version in the URL will show the right thing
+#
+# format documentation:
+# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+# - https://docs.netlify.com/routing/redirects/redirect-options/
+#
+# conventions:
+# - always force (<CODE>!) since this allows re-using file names
+# - group related paths to ease readability
+# - always append new redirects to the end of the file
+# - redirects that should have been there but are missing can be inserted where they belong
+
+/expressions/expression-language /language/ 301!
+/expressions/language-values /language/values 301!
+/expressions/language-constructs /language/constructs 301!
+/expressions/language-operators /language/operators 301!
+/expressions/* /language/:splat 301!
+
+/package-management/basic-package-mgmt /command-ref/nix-env 301!
+
+/package-management/channels* /command-ref/nix-channel 301!
+
+/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301!
+

doc/manual/src/contributing/testing.md

@@ -3,16 +3,28 @@
 ## Unit-tests
 
 The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
-under `src/{library_name}/tests` using the
+under `tests/unit/{library_name}/tests` using the
 [googletest](https://google.github.io/googletest/) and
 [rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
 
 You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`.
 Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable.
 
+### Unit test support libraries
+
+There are headers and code which are not just used to test the library in question, but also downstream libraries.
+For example, we do [property testing] with the [rapidcheck] library.
+This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests.
+Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it.
+Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them.
+
+It is important that these testing libraries don't contain any actual tests themselves.
+On some platforms they would be run as part of every test executable that uses them, which is redundant.
+On other platforms they wouldn't be run at all.
+
 ## Functional tests
 
-The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
+The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
 Each test is a bash script.
 
 ### Running the whole test suite
@@ -21,8 +33,8 @@ The whole test suite can be run with:
 
 ```shell-session
 $ make install && make installcheck
-ran test tests/foo.sh... [PASS]
-ran test tests/bar.sh... [PASS]
+ran test tests/functional/foo.sh... [PASS]
+ran test tests/functional/bar.sh... [PASS]
 ...
 ```
 
@@ -30,14 +42,14 @@ ran test tests/bar.sh... [PASS]
 
 Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
 Each test group is in a subdirectory of `tests`.
-For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
+For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
 
 That test group can be run like this:
 
 ```shell-session
 $ make ca.test-group -j50
-ran test tests/ca/nix-run.sh... [PASS]
-ran test tests/ca/import-derivation.sh... [PASS]
+ran test tests/functional/ca/nix-run.sh... [PASS]
+ran test tests/functional/ca/import-derivation.sh... [PASS]
 ...
 ```
 
@@ -56,21 +68,21 @@ install-tests-groups += $(test-group-name)
 Individual tests can be run with `make`:
 
 ```shell-session
-$ make tests/${testName}.sh.test
-ran test tests/${testName}.sh... [PASS]
+$ make tests/functional/${testName}.sh.test
+ran test tests/functional/${testName}.sh... [PASS]
 ```
 
 or without `make`:
 
 ```shell-session
-$ ./mk/run-test.sh tests/${testName}.sh
-ran test tests/${testName}.sh... [PASS]
+$ ./mk/run-test.sh tests/functional/${testName}.sh
+ran test tests/functional/${testName}.sh... [PASS]
 ```
 
 To see the complete output, one can also run:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh
 + foo
 output from foo
 + bar
@@ -105,7 +117,7 @@ edit it like so:
 Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
 
 ```shell-session
-$ ./mk/debug-test.sh tests/${testName}.sh
+$ ./mk/debug-test.sh tests/functional/${testName}.sh
 ...
 + gdb blash blub
 GNU gdb (GDB) 12.1
@@ -124,9 +136,11 @@ This technique is to include the exact output/behavior of a former version of Ni
 For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered.
 
 It is frequently useful to regenerate the expected output.
-To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`.
-(At least, this is the convention we've used for `tests/lang.sh`.
-If we add more characterization testing we should always strive to be consistent.)
+To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`.
+For example:
+```bash
+_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test
+```
 
 An interesting situation to document is the case when these tests are "overfitted".
 The language tests are, again, an example of this.

doc/manual/src/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="587.11" height="516.604" viewBox="0 0 550.416 484.317"><defs><linearGradient id="a"><stop offset="0" style="stop-color:#699ad7;stop-opacity:1"/><stop offset=".243" style="stop-color:#7eb1dd;stop-opacity:1"/><stop offset="1" style="stop-color:#7ebae4;stop-opacity:1"/></linearGradient><linearGradient id="b"><stop offset="0" style="stop-color:#415e9a;stop-opacity:1"/><stop offset=".232" style="stop-color:#4a6baf;stop-opacity:1"/><stop offset="1" style="stop-color:#5277c3;stop-opacity:1"/></linearGradient><linearGradient xlink:href="#a" id="c" x1="200.597" x2="290.087" y1="351.411" y2="506.188" gradientTransform="translate(70.65 -1055.151)" gradientUnits="userSpaceOnUse"/><linearGradient xlink:href="#b" id="e" x1="-584.199" x2="-496.297" y1="782.336" y2="937.714" gradientTransform="translate(864.696 -1491.34)" gradientUnits="userSpaceOnUse"/></defs><g style="display:inline;opacity:1" transform="translate(-132.651 958.04)"><path id="d" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="opacity:1;fill:url(#c);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(60 407.112 -715.787)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(-60 407.312 -715.7)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(180 407.419 -715.756)"/><path id="f" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="color:#000;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000;solid-opacity:1;fill:url(#e);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(120 407.34 -716.084)"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(-120 407.288 -715.87)"/></g></svg>

doc/manual/src/release-notes/rl-next.md

@@ -1 +1,9 @@
 # Release X.Y (202?-??-??)
+
+- Fix a FOD sandbox escape:
+    Cooperating Nix derivations could send file descriptors to files in the Nix
+    store to each other via Unix domain sockets in the abstract namespace. This
+    allowed one derivation to modify the output of the other derivation, after Nix
+    has registered the path as "valid" and immutable in the Nix database.
+    In particular, this allowed the output of fixed-output derivations to be
+    modified from their expected content. This isn't the case any more.

doc/manual/utils.nix

@@ -44,10 +44,10 @@ rec {
 
   optionalString = cond: string: if cond then string else "";
 
-  showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
+  showSetting = { inlineHTML }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
     let
       result = squash ''
-          - ${if useAnchors
+          - ${if inlineHTML
               then ''<span id="conf-${name}">[`${name}`](#conf-${name})</span>''
               else ''`${name}`''}
 

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1695124524,
-        "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=",
-        "owner": "edolstra",
+        "lastModified": 1700748986,
+        "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a3d30b525535e3158221abc1a957ce798ab159fe",
+        "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
-        "ref": "fix-aws-sdk-cpp",
+        "owner": "NixOS",
+        "ref": "nixos-23.05-small",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,8 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
-  inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
@@ -12,7 +11,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -25,7 +24,7 @@
       linuxSystems = linux32BitSystems ++ linux64BitSystems;
       darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ];
       systems = linuxSystems ++ darwinSystems;
-      
+
       crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
       stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ];
@@ -59,35 +58,28 @@
 
       nixSrc = fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect baseFiles (
-          fileset.difference
-            (fileset.unions [
-              ./.version
-              ./boehmgc-coroutine-sp-fallback.diff
-              ./bootstrap.sh
-              ./configure.ac
-              ./doc
-              ./local.mk
-              ./m4
-              ./Makefile
-              ./Makefile.config.in
-              ./misc
-              ./mk
-              ./precompiled-headers.h
-              ./src
-              ./tests
-              ./COPYING
-              ./scripts/local.mk
-              (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
-              # TODO: do we really need README.md? It doesn't seem used in the build.
-              ./README.md
-            ])
-            (fileset.unions [
-              # Removed file sets
-              ./tests/nixos
-              ./tests/installer
-            ])
-        );
+        fileset = fileset.intersect baseFiles (fileset.unions [
+          ./.version
+          ./boehmgc-coroutine-sp-fallback.diff
+          ./bootstrap.sh
+          ./configure.ac
+          ./doc
+          ./local.mk
+          ./m4
+          ./Makefile
+          ./Makefile.config.in
+          ./misc
+          ./mk
+          ./precompiled-headers.h
+          ./src
+          ./tests/functional
+          ./tests/unit
+          ./COPYING
+          ./scripts/local.mk
+          (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts)
+          # TODO: do we really need README.md? It doesn't seem used in the build.
+          ./README.md
+        ]);
       };
 
       # Memoize nixpkgs for different platforms for efficiency.
@@ -181,7 +173,13 @@
             boost
             lowdown-nix
           ]
-          ++ lib.optionals stdenv.isLinux [libseccomp]
+          ++ lib.optionals stdenv.isLinux [(libseccomp.overrideAttrs (_: rec {
+            version = "2.5.5";
+            src = fetchurl {
+              url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
+              hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
+            };
+          }))]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
           ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
 
@@ -558,7 +556,7 @@
         # tarball for the user's system and calls the second half of the
         # installation script.
         installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "aarch64-darwin" "armv6l-linux" "armv7l-linux"];
 
         # docker image with Nix inside
         dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
@@ -642,6 +640,10 @@
           ["i686-linux" "x86_64-linux"]
           (system: runNixOSTestFor system ./tests/nixos/setuid.nix);
 
+        tests.ca-fd-leak = runNixOSTestFor "x86_64-linux" ./tests/nixos/ca-fd-leak;
+
+        tests.user-sandboxing = runNixOSTestFor "x86_64-linux" ./tests/nixos/user-sandboxing;
+
 
         # Make sure that nix-env still produces the exact same result
         # on a particular version of Nixpkgs.

mk/common-test.sh

@@ -1,11 +1,15 @@
+test_dir=tests/functional
+
+test=$(echo -n "$test" | sed -e "s|^$test_dir/||")
+
 TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=')
 
 : ${BASH:=/usr/bin/env bash}
 
 init_test () {
-   cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
+   cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null
 }
 
 run_test_proper () {
-   cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
+   cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test)
 }

scripts/bigsur-nixbld-user-migration.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-((NEW_NIX_FIRST_BUILD_UID=301))
+((NEW_NIX_FIRST_BUILD_UID=351))
 
 id_available(){
 	dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null

scripts/install-darwin-multi-user.sh

@@ -3,11 +3,24 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+# Notes:
+# - up to macOS Big Sur we used the same GID/UIDs as Linux (30000:30001-32)
+# - we changed UID to 301 because Big Sur updates failed into recovery mode
+#   we're targeting the 200-400 UID range for role users mentioned in the
+#   usage note for sysadminctl
+# - we changed UID to 351 because Sequoia now uses UIDs 300-304 for its own
+#   daemon users
+# - we changed GID to 350 alongside above just because it hides the nixbld
+#   group from the Users & Groups settings panel :)
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-351}"
+export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-350}"
+export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
+
+
 readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
 # create by default; set 0 to DIY, use a symlink, etc.
 readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default
-NIX_FIRST_BUILD_UID="301"
-NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 
 # caution: may update times on / if not run as normal non-root user
 read_only_root() {

scripts/install-multi-user.sh

@@ -23,11 +23,12 @@ readonly RED='\033[31m'
 # installer allows overriding build user count to speed up installation
 # as creating each user takes non-trivial amount of time on macos
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
-readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
-# darwin installer needs to override these
-NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
-NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+# each system specific installer must set these:
+#   NIX_FIRST_BUILD_UID
+#   NIX_BUILD_GROUP_ID
+#   NIX_BUILD_USER_NAME_TEMPLATE
+
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
 readonly NIX_ROOT="/nix"
@@ -522,9 +523,7 @@ It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
 with the UID $primary_group_id. This script can't really handle
 that right now, so I'm going to give up.
 
-You can fix this by editing this script and changing the
-NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
-to $primary_group_id and re-run.
+You can export NIX_BUILD_GROUP_ID=$primary_group_id and re-run.
 EOF
         else
             row "            Exists" "Yes"
@@ -699,6 +698,12 @@ EOF
     fi
 }
 
+check_required_system_specific_settings() {
+    if [ -z "${NIX_FIRST_BUILD_UID+x}" ] || [ -z "${NIX_BUILD_USER_NAME_TEMPLATE+x}" ]; then
+        failure "Internal error: System specific installer for $(uname) ($1) does not export required settings."
+    fi
+}
+
 welcome_to_nix() {
     local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))"
     local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}")
@@ -718,7 +723,9 @@ manager. This will happen in a few stages:
    if you are ready to continue.
 
 3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT}
-   that the Nix daemon uses to run builds.
+   that the Nix daemon uses to run builds. To create system users
+   in a different range, exit and run this tool again with
+   NIX_FIRST_BUILD_UID set.
 
 4. Perform the basic installation of the Nix files daemon.
 
@@ -960,13 +967,16 @@ main() {
     if is_os_darwin; then
         # shellcheck source=./install-darwin-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
+        check_required_system_specific_settings "install-darwin-multi-user.sh"
     elif is_os_linux; then
         # shellcheck source=./install-systemd-multi-user.sh
         . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also
+        check_required_system_specific_settings "install-systemd-multi-user.sh"
     else
         failure "Sorry, I don't know what to do on $(uname)"
     fi
 
+
     welcome_to_nix
 
     if ! is_root; then

scripts/install-systemd-multi-user.sh

@@ -3,6 +3,11 @@
 set -eu
 set -o pipefail
 
+# System specific settings
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
+export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
+export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
+
 readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
 readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 

src/libexpr/eval-cache.cc

@@ -6,6 +6,25 @@
 
 namespace nix::eval_cache {
 
+CachedEvalError::CachedEvalError(ref<EvalState> state, ref<AttrCursor> cursor, Symbol attr)
+    : EvalError("cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
+    , state(state), cursor(cursor), attr(attr)
+{ }
+
+void CachedEvalError::force()
+{
+    auto & v = cursor->forceValue();
+
+    if (v.type() == nAttrs) {
+        auto a = v.attrs->get(this->attr);
+
+        state->forceValue(*a->value, a->pos);
+    }
+
+    // Shouldn't happen.
+    throw EvalError("evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
+}
+
 static const char * schema = R"sql(
 create table if not exists Attributes (
     parent      integer not null,
@@ -469,7 +488,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
     return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
 {
     if (root->db) {
         if (!cachedValue)
@@ -486,12 +505,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
                 if (attr) {
                     if (std::get_if<missing_t>(&attr->second))
                         return nullptr;
-                    else if (std::get_if<failed_t>(&attr->second)) {
-                        if (forceErrors)
-                            debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
-                        else
-                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
-                    } else
+                    else if (std::get_if<failed_t>(&attr->second))
+                        throw CachedEvalError(ref(root->state.shared_from_this()), ref(shared_from_this()), name);
+                    else
                         return std::make_shared<AttrCursor>(root,
                             std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
                 }
@@ -536,9 +552,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name)
 {
-    auto p = maybeGetAttr(name, forceErrors);
+    auto p = maybeGetAttr(name);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
     return ref(p);
@@ -549,11 +565,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
     return getAttr(root->state.symbols.create(name));
 }
 
-OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        auto child = res->maybeGetAttr(attr, force);
+        auto child = res->maybeGetAttr(attr);
         if (!child) {
             auto suggestions = res->getSuggestionsForAttr(attr);
             return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@@ -750,7 +766,7 @@ bool AttrCursor::isDerivation()
 
 StorePath AttrCursor::forceDerivation()
 {
-    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto aDrvPath = getAttr(root->state.sDrvPath);
     auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
     if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
         /* The eval cache contains 'drvPath', but the actual path has

src/libexpr/eval-cache.hh

@@ -10,14 +10,29 @@
 
 namespace nix::eval_cache {
 
-MakeError(CachedEvalError, EvalError);
-
 struct AttrDb;
 class AttrCursor;
 
+struct CachedEvalError : EvalError
+{
+    const ref<EvalState> state;
+    const ref<AttrCursor> cursor;
+    const Symbol attr;
+
+    CachedEvalError(ref<EvalState>, ref<AttrCursor> cursor, Symbol attr);
+
+    /**
+     * Evaluate this attribute, which should result in a regular
+     * `EvalError` exception being thrown.
+     */
+    [[noreturn]]
+    void force();
+};
+
 class EvalCache : public std::enable_shared_from_this<EvalCache>
 {
     friend class AttrCursor;
+    friend class CachedEvalError;
 
     std::shared_ptr<AttrDb> db;
     EvalState & state;
@@ -73,6 +88,7 @@ typedef std::variant<
 class AttrCursor : public std::enable_shared_from_this<AttrCursor>
 {
     friend class EvalCache;
+    friend class CachedEvalError;
 
     ref<EvalCache> root;
     typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@@ -102,11 +118,11 @@ public:
 
     Suggestions getSuggestionsForAttr(Symbol name);
 
-    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name);
 
     ref<AttrCursor> getAttr(std::string_view name);
 
@@ -114,7 +130,7 @@ public:
      * Get an attribute along a chain of attrsets. Note that this does
      * not auto-call functors or functions.
      */
-    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
 
     std::string getString();
 

src/libexpr/eval.hh

@@ -802,7 +802,7 @@ std::string showType(const Value & v);
 /**
  * If `path` refers to a directory, then append "/default.nix".
  */
-SourcePath resolveExprPath(const SourcePath & path);
+SourcePath resolveExprPath(SourcePath path);
 
 struct InvalidPathError : EvalError
 {

src/libexpr/flake/lockfile.cc

@@ -89,7 +89,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
                 std::string inputKey = i.value();
                 auto k = nodeMap.find(inputKey);
                 if (k == nodeMap.end()) {
-                    auto nodes = json["nodes"];
+                    auto & nodes = json["nodes"];
                     auto jsonNode2 = nodes.find(inputKey);
                     if (jsonNode2 == nodes.end())
                         throw Error("lock file references missing node '%s'", inputKey);

src/libexpr/parser.y

@@ -682,17 +682,25 @@ Expr * EvalState::parse(
 }
 
 
-SourcePath resolveExprPath(const SourcePath & path)
+SourcePath resolveExprPath(SourcePath path)
 {
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
-    auto path2 = path.resolveSymlinks();
+    while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
+        if (path.lstat().type != InputAccessor::tSymlink) break;
+        path = {CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))};
+    }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (path2.lstat().type == InputAccessor::tDirectory)
-        return path2 + "default.nix";
+    if (path.lstat().type == InputAccessor::tDirectory)
+        return path + "default.nix";
 
-    return path2;
+    return path;
 }
 
 

src/libexpr/primops.cc

@@ -82,16 +82,15 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->buildPaths(buildReqs);
+    buildStore->buildPaths(buildReqs, bmNormal, store);
+
+    StorePathSet outputsToCopyAndAllow;
 
     for (auto & drv : drvs) {
-        auto outputs = resolveDerivedPath(*store, drv);
+        auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
         for (auto & [outputName, outputPath] : outputs) {
-            /* Add the output of this derivations to the allowed
-               paths. */
-            if (allowedPaths) {
-                allowPath(outputPath);
-            }
+            outputsToCopyAndAllow.insert(outputPath);
+
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
                 res.insert_or_assign(
@@ -100,12 +99,21 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
                             .drvPath = drv.drvPath,
                             .output = outputName,
                         }).render(),
-                    store->printStorePath(outputPath)
+                    buildStore->printStorePath(outputPath)
                 );
             }
         }
     }
 
+    if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
+    if (allowedPaths) {
+        for (auto & outputPath : outputsToCopyAndAllow) {
+            /* Add the output of this derivations to the allowed
+               paths. */
+            allowPath(store->toRealPath(outputPath));
+        }
+    }
+
     return res;
 }
 
@@ -1528,7 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args,
     auto path = realisePath(state, pos, arg, { .checkForPureEval = false });
 
     /* SourcePath doesn't know about trailing slash. */
-    auto mustBeDir = arg.type() == nString && arg.str().ends_with("/");
+    auto mustBeDir = arg.type() == nString
+        && (arg.str().ends_with("/")
+            || arg.str().ends_with("/."));
 
     try {
         auto checked = state.checkSourcePath(path);

src/libexpr/primops/fetchTree.cc

@@ -395,7 +395,8 @@ static RegisterPrimOp primop_fetchGit({
 
       - `shallow` (default: `false`)
 
-        A Boolean parameter that specifies whether fetching a shallow clone is allowed.
+        A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
+        This still performs a full clone of what is available on the remote.
 
       - `allRefs`
 

src/libexpr/tests/local.mk

@@ -1,19 +0,0 @@
-check: libexpr-tests_RUN
-
-programs += libexpr-tests
-
-libexpr-tests_NAME := libnixexpr-tests
-
-libexpr-tests_DIR := $(d)
-
-libexpr-tests_INSTALL_DIR :=
-
-libexpr-tests_SOURCES := \
-    $(wildcard $(d)/*.cc) \
-    $(wildcard $(d)/value/*.cc)
-
-libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers
-
-libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers
-
-libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock

src/libstore/build/create-derivation-and-realise-goal.cc

@@ -1,157 +0,0 @@
-#include "create-derivation-and-realise-goal.hh"
-#include "worker.hh"
-
-namespace nix {
-
-CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
-    const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
-    : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
-    , drvReq(drvReq)
-    , wantedOutputs(wantedOutputs)
-    , buildMode(buildMode)
-{
-    state = &CreateDerivationAndRealiseGoal::getDerivation;
-    name = fmt(
-        "outer obtaining drv from '%s' and then building outputs %s",
-        drvReq->to_string(worker.store),
-        std::visit(overloaded {
-            [&](const OutputsSpec::All) -> std::string {
-                return "* (all of them)";
-            },
-            [&](const OutputsSpec::Names os) {
-                return concatStringsSep(", ", quoteStrings(os));
-            },
-        }, wantedOutputs.raw));
-    trace("created outer");
-
-    worker.updateProgress();
-}
-
-
-CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
-{
-}
-
-
-static StorePath pathPartOfReq(const SingleDerivedPath & req)
-{
-    return std::visit(overloaded {
-        [&](const SingleDerivedPath::Opaque & bo) {
-            return bo.path;
-        },
-        [&](const SingleDerivedPath::Built & bfd) {
-            return pathPartOfReq(*bfd.drvPath);
-        },
-    }, req.raw());
-}
-
-
-std::string CreateDerivationAndRealiseGoal::key()
-{
-    /* Ensure that derivations get built in order of their name,
-       i.e. a derivation named "aardvark" always comes before "baboon". And
-       substitution goals and inner derivation goals always happen before
-       derivation goals (due to "b$"). */
-    return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
-}
-
-
-void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
-{
-}
-
-
-void CreateDerivationAndRealiseGoal::work()
-{
-    (this->*state)();
-}
-
-
-void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
-{
-    /* If we already want all outputs, there is nothing to do. */
-    auto newWanted = wantedOutputs.union_(outputs);
-    bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
-    wantedOutputs = newWanted;
-
-    if (!needRestart) return;
-
-    if (!optDrvPath)
-        // haven't started steps where the outputs matter yet
-        return;
-    worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
-}
-
-
-void CreateDerivationAndRealiseGoal::getDerivation()
-{
-    trace("outer init");
-
-    /* The first thing to do is to make sure that the derivation
-       exists.  If it doesn't, it may be created through a
-       substitute. */
-    if (auto optDrvPath = [this]() -> std::optional<StorePath> {
-        if (buildMode != bmNormal) return std::nullopt;
-
-        auto drvPath = StorePath::dummy;
-        try {
-            drvPath = resolveDerivedPath(worker.store, *drvReq);
-        } catch (MissingRealisation &) {
-            return std::nullopt;
-        }
-        return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
-            ? std::optional { drvPath }
-            : std::nullopt;
-    }()) {
-        trace(fmt("already have drv '%s' for '%s', can go straight to building",
-            worker.store.printStorePath(*optDrvPath),
-            drvReq->to_string(worker.store)));
-
-        loadAndBuildDerivation();
-    } else {
-        trace("need to obtain drv we want to build");
-
-        addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
-
-        state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
-        if (waitees.empty()) work();
-    }
-}
-
-
-void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
-{
-    trace("outer load and build derivation");
-
-    if (nrFailed != 0) {
-        amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
-        return;
-    }
-
-    StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
-    /* Build this step! */
-    concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
-    addWaitee(upcast_goal(concreteDrvGoal));
-    state = &CreateDerivationAndRealiseGoal::buildDone;
-    optDrvPath = std::move(drvPath);
-    if (waitees.empty()) work();
-}
-
-
-void CreateDerivationAndRealiseGoal::buildDone()
-{
-    trace("outer build done");
-
-    buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
-        .drvPath = drvReq,
-        .outputs = wantedOutputs,
-    });
-
-    if (buildResult.success())
-        amDone(ecSuccess);
-    else
-        amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
-}
-
-
-}

src/libstore/build/create-derivation-and-realise-goal.hh

@@ -1,96 +0,0 @@
-#pragma once
-
-#include "parsed-derivations.hh"
-#include "lock.hh"
-#include "store-api.hh"
-#include "pathlocks.hh"
-#include "goal.hh"
-
-namespace nix {
-
-struct DerivationGoal;
-
-/**
- * This goal type is essentially the serial composition (like function
- * composition) of a goal for getting a derivation, and then a
- * `DerivationGoal` using the newly-obtained derivation.
- *
- * In the (currently experimental) general inductive case of derivations
- * that are themselves build outputs, that first goal will be *another*
- * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
- * where the derivation has no provence and is just referred to by
- * (content-addressed) store path, that first goal is a
- * `SubstitutionGoal`.
- *
- * If we already have the derivation (e.g. if the evalutator has created
- * the derivation locally and then instructured the store to build it),
- * we can skip the first goal entirely as a small optimization.
- */
-struct CreateDerivationAndRealiseGoal : public Goal
-{
-    /**
-     * How to obtain a store path of the derivation to build.
-     */
-    ref<SingleDerivedPath> drvReq;
-
-    /**
-     * The path of the derivation, once obtained.
-     **/
-    std::optional<StorePath> optDrvPath;
-
-    /**
-     * The goal for the corresponding concrete derivation.
-     **/
-    std::shared_ptr<DerivationGoal> concreteDrvGoal;
-
-    /**
-     * The specific outputs that we need to build.
-     */
-    OutputsSpec wantedOutputs;
-
-    typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
-    GoalState state;
-
-    /**
-     * The final output paths of the build.
-     *
-     * - For input-addressed derivations, always the precomputed paths
-     *
-     * - For content-addressed derivations, calcuated from whatever the
-     *   hash ends up being. (Note that fixed outputs derivations that
-     *   produce the "wrong" output still install that data under its
-     *   true content-address.)
-     */
-    OutputPathMap finalOutputs;
-
-    BuildMode buildMode;
-
-    CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
-        const OutputsSpec & wantedOutputs, Worker & worker,
-        BuildMode buildMode = bmNormal);
-    virtual ~CreateDerivationAndRealiseGoal();
-
-    void timedOut(Error && ex) override;
-
-    std::string key() override;
-
-    void work() override;
-
-    /**
-     * Add wanted outputs to an already existing derivation goal.
-     */
-    void addWantedOutputs(const OutputsSpec & outputs);
-
-    /**
-     * The states.
-     */
-    void getDerivation();
-    void loadAndBuildDerivation();
-    void buildDone();
-
-    JobCategory jobCategory() const override {
-        return JobCategory::Administration;
-    };
-};
-
-}

src/libstore/build/derivation-goal.cc

@@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
     , wantedOutputs(wantedOutputs)
     , buildMode(buildMode)
 {
-    state = &DerivationGoal::loadDerivation;
+    state = &DerivationGoal::getDerivation;
     name = fmt(
         "building of '%s' from .drv file",
         DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -164,6 +164,24 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
 }
 
 
+void DerivationGoal::getDerivation()
+{
+    trace("init");
+
+    /* The first thing to do is to make sure that the derivation
+       exists.  If it doesn't, it may be created through a
+       substitute. */
+    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
+        loadDerivation();
+        return;
+    }
+
+    addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
+
+    state = &DerivationGoal::loadDerivation;
+}
+
+
 void DerivationGoal::loadDerivation()
 {
     trace("loading derivation");
@@ -1498,24 +1516,23 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
     if (!useDerivation) return;
     auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-    std::optional info = tryGetConcreteDrvGoal(waitee);
-    if (!info) return;
-    const auto & [dg, drvReq] = *info;
+    auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
+    if (!dg) return;
 
-    auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get());
+    auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath });
     if (!nodeP) return;
     auto & outputs = nodeP->value;
 
     for (auto & outputName : outputs) {
-        auto buildResult = dg.get().getBuildResult(DerivedPath::Built {
-            .drvPath = makeConstantStorePathRef(dg.get().drvPath),
+        auto buildResult = dg->getBuildResult(DerivedPath::Built {
+            .drvPath = makeConstantStorePathRef(dg->drvPath),
             .outputs = OutputsSpec::Names { outputName },
         });
         if (buildResult.success()) {
             auto i = buildResult.builtOutputs.find(outputName);
             if (i != buildResult.builtOutputs.end())
                 inputDrvOutputs.insert_or_assign(
-                    { dg.get().drvPath, outputName },
+                    { dg->drvPath, outputName },
                     i->second.outPath);
         }
     }

src/libstore/build/derivation-goal.hh

@@ -52,10 +52,6 @@ struct InitialOutput {
 
 /**
  * A goal for building some or all of the outputs of a derivation.
- *
- * The derivation must already be present, either in the store in a drv
- * or in memory. If the derivation itself needs to be gotten first, a
- * `CreateDerivationAndRealiseGoal` goal must be used instead.
  */
 struct DerivationGoal : public Goal
 {
@@ -235,6 +231,7 @@ struct DerivationGoal : public Goal
     /**
      * The states.
      */
+    void getDerivation();
     void loadDerivation();
     void haveDerivation();
     void outputsSubstitutionTried();

src/libstore/build/entry-points.cc

@@ -1,6 +1,5 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
-#include "create-derivation-and-realise-goal.hh"
 #include "derivation-goal.hh"
 #include "local-store.hh"
 
@@ -16,7 +15,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     worker.run(goals);
 
-    StringSet failed;
+    StorePathSet failed;
     std::optional<Error> ex;
     for (auto & i : goals) {
         if (i->ex) {
@@ -26,10 +25,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
                 ex = std::move(i->ex);
         }
         if (i->exitCode != Goal::ecSuccess) {
-            if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
-                failed.insert(i2->drvReq->to_string(*this));
+            if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
+                failed.insert(i2->drvPath);
             else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
-                failed.insert(printStorePath(i2->storePath));
+                failed.insert(i2->storePath);
         }
     }
 
@@ -38,7 +37,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
         throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
-        throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
+        throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
     }
 }
 

src/libstore/build/goal.hh

@@ -49,16 +49,6 @@ enum struct JobCategory {
      * A substitution an arbitrary store object; it will use network resources.
      */
     Substitution,
-    /**
-     * A goal that does no "real" work by itself, and just exists to depend on
-     * other goals which *do* do real work. These goals therefore are not
-     * limited.
-     *
-     * These goals cannot infinitely create themselves, so there is no risk of
-     * a "fork bomb" type situation (which would be a problem even though the
-     * goal do no real work) either.
-     */
-    Administration,
 };
 
 struct Goal : public std::enable_shared_from_this<Goal>

src/libstore/build/local-derivation-goal.cc

@@ -16,6 +16,7 @@
 #include "cgroup.hh"
 #include "personality.hh"
 #include "namespaces.hh"
+#include "file-system.hh"
 
 #include <regex>
 #include <queue>
@@ -35,6 +36,7 @@
 /* Includes required for chroot support. */
 #if __linux__
 #include <sys/ioctl.h>
+#include "linux/fchmodat2-compat.hh"
 #include <net/if.h>
 #include <netinet/ip.h>
 #include <sys/mman.h>
@@ -170,6 +172,10 @@ void LocalDerivationGoal::killSandbox(bool getStats)
 
 void LocalDerivationGoal::tryLocalBuild()
 {
+#if __APPLE__
+    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
+#endif
+
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
         state = &DerivationGoal::tryToBuild;
@@ -476,14 +482,26 @@ void LocalDerivationGoal::startBuilder()
             settings.thisSystem,
             concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
 
-#if __APPLE__
-    additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
     /* Create a temporary directory where the build will take
        place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+    topTmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+#if __APPLE__
+    if (false) {
+#else
+    if (useChroot) {
+#endif
+        /* If sandboxing is enabled, put the actual TMPDIR underneath
+           an inaccessible root-owned directory, to prevent outside
+           access.
 
+           On macOS, we don't use an actual chroot, so this isn't
+           possible. Any mitigation along these lines would have to be
+           done directly in the sandbox profile. */
+        tmpDir = topTmpDir + "/build";
+        createDir(tmpDir, 0700);
+    } else {
+        tmpDir = topTmpDir;
+    }
     chownToBuilder(tmpDir);
 
     for (auto & [outputName, status] : initialOutputs) {
@@ -649,17 +667,21 @@ void LocalDerivationGoal::startBuilder()
 #if __linux__
         /* Create a temporary directory in which we set up the chroot
            environment using bind-mounts.  We put it in the Nix store
-           to ensure that we can create hard-links to non-directory
-           inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
-        deletePath(chrootRootDir);
+           so that the build outputs can be moved efficiently from the
+           chroot to their final location. */
+        chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
+        deletePath(chrootParentDir);
 
         /* Clean up the chroot directory automatically. */
-        autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
+        autoDelChroot = std::make_shared<AutoDelete>(chrootParentDir);
 
-        printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir);
+        printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir);
+
+        if (mkdir(chrootParentDir.c_str(), 0700) == -1)
+            throw SysError("cannot create '%s'", chrootRootDir);
+
+        chrootRootDir = chrootParentDir + "/root";
 
-        // FIXME: make this 0700
         if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
@@ -1662,6 +1684,10 @@ void setupSeccomp()
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
                 SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
             throw SysError("unable to add seccomp rule");
+
+        if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1,
+                SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+            throw SysError("unable to add seccomp rule");
     }
 
     /* Prevent builders from creating EAs or ACLs. Not all filesystems
@@ -2111,10 +2137,11 @@ void LocalDerivationGoal::runChild()
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
                to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
-            Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true);
+            Path globalTmpDir = canonPath(defaultTempDir(), true);
 
             /* They don't like trailing slashes on subpath directives */
-            if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
+            while (!globalTmpDir.empty() && globalTmpDir.back() == '/')
+                globalTmpDir.pop_back();
 
             if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
                 builder = "/usr/bin/sandbox-exec";
@@ -2558,6 +2585,12 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             [&](const DerivationOutput::CAFixed & dof) {
                 auto & wanted = dof.ca.hash;
 
+                // Replace the output by a fresh copy of itself to make sure
+                // that there's no stale file descriptor pointing to it
+                Path tmpOutput = actualPath + ".tmp";
+                copyFile(actualPath, tmpOutput, true);
+                renameFile(tmpOutput, actualPath);
+
                 auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
                     .method = dof.ca.method,
                     .hashType = wanted.type,
@@ -2934,15 +2967,17 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
 
 void LocalDerivationGoal::deleteTmpDir(bool force)
 {
-    if (tmpDir != "") {
+    if (topTmpDir != "") {
         /* Don't keep temporary directories for builtins because they
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv->isBuiltin()) {
             printError("note: keeping build directory '%s'", tmpDir);
+            chmod(topTmpDir.c_str(), 0755);
             chmod(tmpDir.c_str(), 0755);
         }
         else
-            deletePath(tmpDir);
+            deletePath(topTmpDir);
+        topTmpDir = "";
         tmpDir = "";
     }
 }

src/libstore/build/local-derivation-goal.hh

@@ -26,10 +26,16 @@ struct LocalDerivationGoal : public DerivationGoal
     std::optional<Path> cgroup;
 
     /**
-     * The temporary directory.
+     * The temporary directory used for the build.
      */
     Path tmpDir;
 
+    /**
+     * The top-level temporary directory. `tmpDir` is either equal to
+     * or a child of this directory.
+     */
+    Path topTmpDir;
+
     /**
      * The path of the temporary directory in the sandbox.
      */
@@ -64,6 +70,16 @@ struct LocalDerivationGoal : public DerivationGoal
      */
     bool useChroot = false;
 
+    /**
+     * The parent directory of `chrootRootDir`. It has permission 700
+     * and is owned by root to ensure other users cannot mess with
+     * `chrootRootDir`.
+     */
+    Path chrootParentDir;
+
+    /**
+     * The root of the chroot environment.
+     */
     Path chrootRootDir;
 
     /**

src/libstore/build/worker.cc

@@ -2,7 +2,6 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "drv-output-substitution-goal.hh"
-#include "create-derivation-and-realise-goal.hh"
 #include "local-derivation-goal.hh"
 #include "hook-instance.hh"
 
@@ -42,24 +41,6 @@ Worker::~Worker()
 }
 
 
-std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
-    ref<SingleDerivedPath> drvReq,
-    const OutputsSpec & wantedOutputs,
-    BuildMode buildMode)
-{
-    std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
-    std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
-    if (!goal) {
-        goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
-        goal_weak = goal;
-        wakeUp(goal);
-    } else {
-        goal->addWantedOutputs(wantedOutputs);
-    }
-    return goal;
-}
-
-
 std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
@@ -130,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
         [&](const DerivedPath::Built & bfd) -> GoalPtr {
-            return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
+            if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
+                return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
+            else
+                throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
         [&](const DerivedPath::Opaque & bo) -> GoalPtr {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@@ -139,46 +123,24 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 }
 
 
-template<typename K, typename V, typename F>
-static void cullMap(std::map<K, V> & goalMap, F f)
-{
-    for (auto i = goalMap.begin(); i != goalMap.end();)
-        if (!f(i->second))
-            i = goalMap.erase(i);
-        else ++i;
-}
-
-
 template<typename K, typename G>
 static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
 {
     /* !!! inefficient */
-    cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
-        return gp.lock() != goal;
-    });
-}
-
-template<typename K>
-static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
-
-template<typename K>
-static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
-{
-    /* !!! inefficient */
-    cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
-        if (node.value.lock() == goal)
-            node.value.reset();
-        removeGoal(goal, node.childMap);
-        return !node.value.expired() || !node.childMap.empty();
-    });
+    for (auto i = goalMap.begin();
+         i != goalMap.end(); )
+        if (i->second.lock() == goal) {
+            auto j = i; ++j;
+            goalMap.erase(i);
+            i = j;
+        }
+        else ++i;
 }
 
 
 void Worker::removeGoal(GoalPtr goal)
 {
-    if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
-        nix::removeGoal(drvGoal, outerDerivationGoals.map);
-    else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
+    if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
         nix::removeGoal(drvGoal, derivationGoals);
     else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
         nix::removeGoal(subGoal, substitutionGoals);
@@ -236,19 +198,8 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
     child.respectTimeouts = respectTimeouts;
     children.emplace_back(child);
     if (inBuildSlot) {
-        switch (goal->jobCategory()) {
-        case JobCategory::Substitution:
-            nrSubstitutions++;
-            break;
-        case JobCategory::Build:
-            nrLocalBuilds++;
-            break;
-        case JobCategory::Administration:
-            /* Intentionally not limited, see docs */
-            break;
-        default:
-            abort();
-        }
+        if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
+        else nrLocalBuilds++;
     }
 }
 
@@ -260,20 +211,12 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
     if (i == children.end()) return;
 
     if (i->inBuildSlot) {
-        switch (goal->jobCategory()) {
-        case JobCategory::Substitution:
+        if (goal->jobCategory() == JobCategory::Substitution) {
             assert(nrSubstitutions > 0);
             nrSubstitutions--;
-            break;
-        case JobCategory::Build:
+        } else {
             assert(nrLocalBuilds > 0);
             nrLocalBuilds--;
-            break;
-        case JobCategory::Administration:
-            /* Intentionally not limited, see docs */
-            break;
-        default:
-            abort();
         }
     }
 
@@ -324,9 +267,9 @@ void Worker::run(const Goals & _topGoals)
 
     for (auto & i : _topGoals) {
         topGoals.insert(i);
-        if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
+        if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
             topPaths.push_back(DerivedPath::Built {
-                .drvPath = goal->drvReq,
+                .drvPath = makeConstantStorePathRef(goal->drvPath),
                 .outputs = goal->wantedOutputs,
             });
         } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@@ -589,19 +532,4 @@ GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
     return subGoal;
 }
 
-GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
-{
-    return subGoal;
-}
-
-std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee)
-{
-    auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
-    if (!odg) return std::nullopt;
-    return {{
-        std::cref(*odg->concreteDrvGoal),
-        std::cref(*odg->drvReq),
-    }};
-}
-
 }

src/libstore/build/worker.hh

@@ -4,7 +4,6 @@
 #include "types.hh"
 #include "lock.hh"
 #include "store-api.hh"
-#include "derived-path-map.hh"
 #include "goal.hh"
 #include "realisation.hh"
 
@@ -14,7 +13,6 @@
 namespace nix {
 
 /* Forward definition. */
-struct CreateDerivationAndRealiseGoal;
 struct DerivationGoal;
 struct PathSubstitutionGoal;
 class DrvOutputSubstitutionGoal;
@@ -33,25 +31,9 @@ class DrvOutputSubstitutionGoal;
  */
 GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
 GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
-GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
 
 typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
-/**
- * The current implementation of impure derivations has
- * `DerivationGoal`s accumulate realisations from their waitees.
- * Unfortunately, `DerivationGoal`s don't directly depend on other
- * goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
- *
- * We try not to share any of the details of any goal type with any
- * other, for sake of modularity and quicker rebuilds. This means we
- * cannot "just" downcast and fish out the field. So as an escape hatch,
- * we have made the function, written in `worker.cc` where all the goal
- * types are visible, and use it instead.
- */
-
-std::optional<std::pair<std::reference_wrapper<const DerivationGoal>, std::reference_wrapper<const SingleDerivedPath>>> tryGetConcreteDrvGoal(GoalPtr waitee);
-
 /**
  * A mapping used to remember for each child process to what goal it
  * belongs, and file descriptors for receiving log data and output
@@ -119,9 +101,6 @@ private:
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
-
-    DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
-
     std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
     std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
     std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@@ -209,9 +188,6 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
-    std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
-        ref<SingleDerivedPath> drvPath,
-        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
     std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);

src/libstore/builtins/fetchurl.cc

@@ -34,10 +34,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
         auto source = sinkToSource([&](Sink & sink) {
 
-            /* No need to do TLS verification, because we check the hash of
-               the result anyway. */
             FileTransferRequest request(url);
-            request.verifyTLS = false;
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(

src/libstore/derived-path-map.cc

@@ -51,11 +51,8 @@ typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const Single
 
 // instantiations
 
-#include "create-derivation-and-realise-goal.hh"
 namespace nix {
 
-template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
-
 GENERATE_CMP_EXT(
     template<>,
     DerivedPathMap<std::set<std::string>>::ChildNode,

src/libstore/derived-path-map.hh

@@ -20,11 +20,8 @@ namespace nix {
  *
  * @param V A type to instantiate for each output. It should probably
  * should be an "optional" type so not every interior node has to have a
- * value. For example, the scheduler uses
- * `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
- * remember which goals correspond to which outputs. `* const Something`
- * or `std::optional<Something>` would also be good choices for
- * "optional" types.
+ * value. `* const Something` or `std::optional<Something>` would be
+ * good choices for "optional" types.
  */
 template<typename V>
 struct DerivedPathMap {

src/libstore/gc.cc

@@ -776,7 +776,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         }
     };
 
-    /* Synchronisation point for testing, see tests/gc-concurrent.sh. */
+    /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */
     if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
         readFile(*p);
 

src/libstore/globals.cc

@@ -4,6 +4,7 @@
 #include "args.hh"
 #include "abstract-setting-to-json.hh"
 #include "compute-levels.hh"
+#include "file-system.hh"
 
 #include <algorithm>
 #include <map>
@@ -24,6 +25,9 @@
 
 #include "config-impl.hh"
 
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#endif
 
 namespace nix {
 
@@ -154,6 +158,29 @@ unsigned int Settings::getDefaultCores()
       return concurrency;
 }
 
+#if __APPLE__
+static bool hasVirt() {
+
+    int hasVMM;
+    int hvSupport;
+    size_t size;
+
+    size = sizeof(hasVMM);
+    if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) {
+        if (hasVMM)
+            return false;
+    }
+
+    // whether the kernel and hardware supports virt
+    size = sizeof(hvSupport);
+    if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) {
+        return hvSupport == 1;
+    } else {
+        return false;
+    }
+}
+#endif
+
 StringSet Settings::getDefaultSystemFeatures()
 {
     /* For backwards compatibility, accept some "features" that are
@@ -170,6 +197,11 @@ StringSet Settings::getDefaultSystemFeatures()
         features.insert("kvm");
     #endif
 
+    #if __APPLE__
+    if (hasVirt())
+        features.insert("apple-virt");
+    #endif
+
     return features;
 }
 
@@ -377,7 +409,7 @@ void initLibStore() {
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
+    if (hasPrefix(defaultTempDir(), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 

src/libstore/globals.hh

@@ -708,6 +708,7 @@ public:
           `kvm` feature.
 
           This setting by default includes `kvm` if `/dev/kvm` is accessible,
+          `apple-virt` if hardware virtualization is available on macOS,
           and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
           that are used in Nixpkgs to route builds to specific machines.
         )", {}, false};

src/libstore/http-binary-cache-store.cc

@@ -49,7 +49,7 @@ public:
         , BinaryCacheStore(params)
         , cacheUri(scheme + "://" + _cacheUri)
     {
-        if (cacheUri.back() == '/')
+        while (!cacheUri.empty() && cacheUri.back() == '/')
             cacheUri.pop_back();
 
         diskCache = getNarInfoDiskCache();

src/libstore/linux/fchmodat2-compat.hh

@@ -0,0 +1,34 @@
+/*
+ * Determine the syscall number for `fchmodat2`.
+ *
+ * On most platforms this is 452. Exceptions can be found on
+ * a glibc git checkout via `rg --pcre2 'define __NR_fchmodat2 (?!452)'`.
+ *
+ * The problem is that glibc 2.39 and libseccomp 2.5.5 are needed to
+ * get the syscall number. However, a Nix built against nixpkgs 23.11
+ * (glibc 2.38) should still have the issue fixed without depending
+ * on the build environment.
+ *
+ * To achieve that, the macros below try to determine the platform and
+ * set the syscall number which is platform-specific, but
+ * in most cases 452.
+ *
+ * TODO: remove this when 23.11 is EOL and the entire (supported) ecosystem
+ * is on glibc 2.39.
+ */
+
+#if HAVE_SECCOMP
+#  if defined(__alpha__)
+#    define NIX_SYSCALL_FCHMODAT2 562
+#  elif defined(__x86_64__) && SIZE_MAX == 0xFFFFFFFF // x32
+#    define NIX_SYSCALL_FCHMODAT2 1073742276
+#  elif defined(__mips__) && defined(__mips64) && defined(_ABIN64) // mips64/n64
+#    define NIX_SYSCALL_FCHMODAT2 5452
+#  elif defined(__mips__) && defined(__mips64) && defined(_ABIN32) // mips64/n32
+#    define NIX_SYSCALL_FCHMODAT2 6452
+#  elif defined(__mips__) && defined(_ABIO32) // mips32
+#    define NIX_SYSCALL_FCHMODAT2 4452
+#  else
+#    define NIX_SYSCALL_FCHMODAT2 452
+#  endif
+#endif // HAVE_SECCOMP

src/libstore/path-regex.hh

@@ -3,6 +3,11 @@
 
 namespace nix {
 
-static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)";
+
+static constexpr std::string_view nameRegexStr =
+    // This uses a negative lookahead: (?!\.\.?(-|$))
+    //   - deny ".", "..", or those strings followed by '-'
+    //   - when it's not those, start again at the start of the input and apply the next regex, which is [0-9a-zA-Z\+\-\._\?=]+
+    R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)";
 
 }

src/libstore/path.cc

@@ -12,6 +12,19 @@ static void checkName(std::string_view path, std::string_view name)
         throw BadStorePath("store path '%s' has a name longer than %d characters",
             path, StorePath::MaxPathLen);
     // See nameRegexStr for the definition
+    if (name[0] == '.') {
+        // check against "." and "..", followed by end or dash
+        if (name.size() == 1)
+            throw BadStorePath("store path '%s' has invalid name '%s'", path, name);
+        if (name[1] == '-')
+            throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".");
+        if (name[1] == '.') {
+            if (name.size() == 2)
+                throw BadStorePath("store path '%s' has invalid name '%s'", path, name);
+            if (name[2] == '-')
+                throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "..");
+        }
+    }
     for (auto c : name)
         if (!((c >= '0' && c <= '9')
                 || (c >= 'a' && c <= 'z')

src/libstore/tests/local.mk

@@ -1,29 +0,0 @@
-check: libstore-tests-exe_RUN
-
-programs += libstore-tests-exe
-
-libstore-tests-exe_NAME = libnixstore-tests
-
-libstore-tests-exe_DIR := $(d)
-
-libstore-tests-exe_INSTALL_DIR :=
-
-libstore-tests-exe_LIBS = libstore-tests
-
-libstore-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libstore-tests
-
-libstore-tests_NAME = libnixstore-tests
-
-libstore-tests_DIR := $(d)
-
-libstore-tests_INSTALL_DIR :=
-
-libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
-
-libstore-tests_LIBS = libutil-tests libstore libutil
-
-libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

src/libutil/file-system.hh

@@ -0,0 +1,17 @@
+#pragma once
+/**
+ * @file
+ *
+ * Utiltities for working with the file sytem and file paths.
+ */
+
+#include "types.hh"
+
+namespace nix {
+
+/**
+ * Return `TMPDIR`, or the default temporary directory if unset or empty.
+ */
+Path defaultTempDir();
+
+}

src/libutil/filesystem.cc

@@ -5,15 +5,20 @@
 #include "finally.hh"
 #include "util.hh"
 #include "types.hh"
+#include "file-system.hh"
 
 namespace fs = std::filesystem;
 
 namespace nix {
 
+std::string defaultTempDir() {
+    return getEnvNonEmpty("TMPDIR").value_or("/tmp");
+}
+
 static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
     std::atomic<unsigned int> & counter)
 {
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    tmpRoot = canonPath(tmpRoot.empty() ? defaultTempDir() : tmpRoot, true);
     if (includePid)
         return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++);
     else
@@ -53,7 +58,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 
 std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 {
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX");
     // Strictly speaking, this is UB, but who cares...
     // FIXME: use O_TMPFILE.
     AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
@@ -133,6 +138,12 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
     }
 }
 
+
+void copyFile(const Path & oldPath, const Path & newPath, bool andDelete)
+{
+    return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), andDelete);
+}
+
 void renameFile(const Path & oldName, const Path & newName)
 {
     fs::rename(oldName, newName);

src/libutil/serialise.cc

@@ -444,7 +444,7 @@ Error readError(Source & source)
     auto msg = readString(source);
     ErrorInfo info {
         .level = level,
-        .msg = hintformat(fmt("%s", msg)),
+        .msg = hintfmt(msg),
     };
     auto havePos = readNum<size_t>(source);
     assert(havePos == 0);
@@ -453,7 +453,7 @@ Error readError(Source & source)
         havePos = readNum<size_t>(source);
         assert(havePos == 0);
         info.traces.push_back(Trace {
-            .hint = hintformat(fmt("%s", readString(source)))
+            .hint = hintfmt(readString(source))
         });
     }
     return Error(std::move(info));

src/libutil/tests/local.mk

@@ -1,29 +0,0 @@
-check: libutil-tests_RUN
-
-programs += libutil-tests
-
-libutil-tests-exe_NAME = libnixutil-tests
-
-libutil-tests-exe_DIR := $(d)
-
-libutil-tests-exe_INSTALL_DIR :=
-
-libutil-tests-exe_LIBS = libutil-tests
-
-libutil-tests-exe_LDFLAGS := $(GTEST_LIBS)
-
-libraries += libutil-tests
-
-libutil-tests_NAME = libnixutil-tests
-
-libutil-tests_DIR := $(d)
-
-libutil-tests_INSTALL_DIR :=
-
-libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
-
-libutil-tests_CXXFLAGS += -I src/libutil
-
-libutil-tests_LIBS = libutil
-
-libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS)

src/libutil/url-parts.hh

@@ -30,7 +30,7 @@ extern std::regex refRegex;
 
 /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
 /// This is because of the definition of a ref in refs.c in https://github.com/git/git
-/// See tests/fetchGitRefs.sh for the full definition
+/// See tests/functional/fetchGitRefs.sh for the full definition
 const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$";
 extern std::regex badGitRefRegex;
 

src/libutil/util.cc

@@ -679,6 +679,11 @@ std::optional<Path> getSelfExe()
     return cached;
 }
 
+void createDir(const Path &path, mode_t mode)
+{
+    if (mkdir(path.c_str(), mode) == -1)
+        throw SysError("creating directory '%1%'", path);
+}
 
 Paths createDirs(const Path & path)
 {

src/libutil/util.hh

@@ -253,6 +253,11 @@ inline Paths createDirs(PathView path)
     return createDirs(Path(path));
 }
 
+/**
+ * Create a single directory.
+ */
+void createDir(const Path & path, mode_t mode = 0755);
+
 /**
  * Create a symlink.
  */
@@ -274,6 +279,13 @@ void renameFile(const Path & src, const Path & dst);
  */
 void moveFile(const Path & src, const Path & dst);
 
+/**
+ * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is
+ * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but
+ * with the guaranty that the destination will be “fresh”, with no stale inode
+ * or file descriptor pointing to it).
+ */
+void copyFile(const Path & oldPath, const Path & newPath, bool andDelete);
 
 /**
  * Wrappers arount read()/write() that read/write exactly the

src/nix-build/nix-build.cc

@@ -119,7 +119,7 @@ static void main_nix_build(int argc, char * * argv)
         script = argv[1];
         try {
             auto lines = tokenizeString<Strings>(readFile(script), "\n");
-            if (std::regex_search(lines.front(), std::regex("^#!"))) {
+            if (!lines.empty() && std::regex_search(lines.front(), std::regex("^#!"))) {
                 lines.pop_front();
                 inShebang = true;
                 for (int i = 2; i < argc; ++i)
@@ -457,8 +457,7 @@ static void main_nix_build(int argc, char * * argv)
         // Set the environment.
         auto env = getEnv();
 
-        auto tmp = getEnv("TMPDIR");
-        if (!tmp) tmp = getEnv("XDG_RUNTIME_DIR").value_or("/tmp");
+        auto tmp = getEnvNonEmpty("TMPDIR").value_or("/tmp");
 
         if (pure) {
             decltype(env) newEnv;
@@ -470,7 +469,7 @@ static void main_nix_build(int argc, char * * argv)
             env["__ETC_PROFILE_SOURCED"] = "1";
         }
 
-        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp;
+        env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp;
         env["NIX_STORE"] = store->storeDir;
         env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
 

src/nix/daemon.cc

@@ -201,7 +201,11 @@ static PeerInfo getPeerInfo(int remote)
 
 #if defined(SO_PEERCRED)
 
-    ucred cred;
+# if defined(__OpenBSD__)
+   struct sockpeercred cred;
+# else
+   ucred cred;
+# endif
     socklen_t credLen = sizeof(cred);
     if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1)
         throw SysError("getting peer credentials");
@@ -209,9 +213,9 @@ static PeerInfo getPeerInfo(int remote)
 
 #elif defined(LOCAL_PEERCRED)
 
-#if !defined(SOL_LOCAL)
-#define SOL_LOCAL 0
-#endif
+# if !defined(SOL_LOCAL)
+# define SOL_LOCAL 0
+# endif
 
     xucred cred;
     socklen_t credLen = sizeof(cred);

src/nix/flake.cc

@@ -1110,7 +1110,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON
                 // If we don't recognize it, it's probably content
                 return true;
             } catch (EvalError & e) {
-                // Some attrs may contain errors, eg. legacyPackages of
+                // Some attrs may contain errors, e.g. legacyPackages of
                 // nixpkgs. We still want to recurse into it, instead of
                 // skipping it at all.
                 return true;

src/nix/main.cc

@@ -12,6 +12,7 @@
 #include "finally.hh"
 #include "loggers.hh"
 #include "markdown.hh"
+#include "eval-cache.hh"
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -218,7 +219,8 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
     vDump->mkString(toplevel.dumpCli());
 
     auto vRes = state.allocValue();
-    state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos);
+    state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos);
+    state.callFunction(*vRes, *vDump, *vRes, noPos);
 
     auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
     if (!attr)
@@ -479,7 +481,15 @@ void mainWrapped(int argc, char * * argv)
     if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) {
         evalSettings.pureEval = false;
     }
-    args.command->second->run();
+
+    try {
+        args.command->second->run();
+    } catch (eval_cache::CachedEvalError & e) {
+        /* Evaluate the original attribute that resulted in this
+           cached error so that we can show the original error to the
+           user. */
+        e.force();
+    }
 }
 
 }

tests/dyn-drv/dep-built-drv.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-source common.sh
-
-out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
-
-clearStore
-
-out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link)
-
-diff -r $out1 $out2

tests/functional/bash-profile.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
+sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
 
 user=$(whoami)
 rm -rf $TEST_HOME $TEST_ROOT/profile-var

tests/functional/build-remote-trustless.sh

@@ -6,7 +6,7 @@ unset NIX_STATE_DIR
 
 remoteDir=$TEST_ROOT/remote
 
-# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for
+# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for
 # more details.
 nix-build $file -o $TEST_ROOT/result --max-jobs 0 \
   --arg busybox $busybox \