set up builder pipe

flake.lock: Update

.github/workflows/test.yml

@@ -20,8 +20,7 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    #- run: nix flake check
-    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest

benchmark.sh

@@ -1,53 +0,0 @@
-#!/bin/sh
-
-set -euo pipefail
-set -x
-
-callNix () {
-    nix \
-        --experimental-features "nix-command flakes" \
-        --store /tmp/nix \
-        "$@"
-}
-
-callBuild () {
-    callNix \
-        eval --impure --file "$THINGTOBENCH" drvPath \
-        "$@"
-}
-getCompletions () {
-    NIX_GET_COMPLETIONS=6 callNix build "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31#firef" "$@"
-}
-runSearch () {
-    callNix search "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31" firefox "$@"
-}
-
-noCache () {
-    "$@" --option eval-cache false
-}
-coldCache () {
-    if [[ -e ~/.cache/nix/eval-cache-v2 || -e ~/.cache/nix/eval-cache-v3 ]]; then
-        echo "Error: The cache should be clean"
-        exit 1
-    fi
-    "$@"
-}
-
-run_all () {
-
-    mkdir -p "$out"
-
-    NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$out/eval-stats.json bash $0 noCache callBuild
-
-    hyperfine \
-        --warmup 2 \
-        --export-csv "$out/result.csv" \
-        --export-json "$out/result.json" \
-        --export-markdown "$out/result.md" \
-        --style basic \
-        --prepare '' "bash $0 noCache callBuild" \
-        --prepare 'rm -rf ~/.cache/nix/' "bash $0 coldCache callBuild" \
-        --prepare '' "bash $0 callBuild"
-}
-
-"$@"

configure.ac

@@ -65,7 +65,7 @@ AC_SYS_LARGEFILE
 AC_STRUCT_DIRENT_D_TYPE
 if test "$sys_name" = sunos; then
     # Solaris requires -lsocket -lnsl for network functions
-    LIBS="-lsocket -lnsl $LIBS"
+    LDFLAGS="-lsocket -lnsl $LDFLAGS"
 fi
 
 
@@ -150,7 +150,7 @@ int main() {
 }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
 AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
 if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
-    LIBS="-latomic $LIBS"
+    LDFLAGS="-latomic $LDFLAGS"
 fi
 
 PKG_PROG_PKG_CONFIG

doc/manual/src/command-ref/env-common.md

@@ -10,35 +10,39 @@ Most Nix commands interpret the following environment variables:
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
-    
+
         /home/eelco/Dev:/etc/nixos
-    
+
     will cause Nix to look for paths relative to `/home/eelco/Dev` and
     `/etc/nixos`, in this order. It is also possible to match paths
     against a prefix. For example, the value
-    
+
         nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
-    
+
     will cause Nix to search for `<nixpkgs/path>` in
     `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-    
+
     If a path in the Nix search path starts with `http://` or
     `https://`, it is interpreted as the URL of a tarball that will be
     downloaded and unpacked to a temporary location. The tarball must
     consist of a single top-level directory. For example, setting
     `NIX_PATH` to
-    
-        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
-    
-    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
-    channel.
-    
-    A following shorthand can be used to refer to the official channels:
-    
-        nixpkgs=channel:nixos-15.09
-    
-    The search path can be extended using the `-I` option, which takes
-    precedence over `NIX_PATH`.
+
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
+
+    tells Nix to download and use the current contents of the
+    `master` branch in the `nixpkgs` repository.
+
+    The URLs of the tarballs from the official nixos.org channels (see
+    [the manual for `nix-channel`](nix-channel.md)) can be abbreviated
+    as `channel:<channel-name>`.  For instance, the following two
+    values of `NIX_PATH` are equivalent:
+
+        nixpkgs=channel:nixos-21.05
+        nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
+
+    The Nix search path can also be extended using the `-I` option to
+    many Nix commands, which takes precedence over `NIX_PATH`.
 
   - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
@@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables:
     builds are deployed to machines where `/nix/store` resolves
     differently. If you are sure that you’re not going to do that, you
     can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
-    
+
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
@@ -59,7 +63,7 @@ Most Nix commands interpret the following environment variables:
     $ mkdir /nix
     $ mount -o bind /mnt/otherdisk/nix /nix
     ```
-    
+
     Consult the mount 8 manual page for details.
 
   - `NIX_STORE_DIR`\

flake.lock

@@ -19,11 +19,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1622593737,
-        "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=",
+        "lastModified": 1624862269,
+        "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e",
+        "rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
         "type": "github"
       },
       "original": {

flake.nix

@@ -229,7 +229,7 @@
             echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
           '';
 
-          doInstallCheck = false;
+          doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           separateDebugInfo = true;
@@ -256,7 +256,8 @@
                 boost
                 nlohmann_json
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -294,13 +295,6 @@
             '';
         };
 
-        nix-benchmarks = prev.runCommandNoCC "nix-benchmarks" {
-          buildInputs = [ final.nix prev.hyperfine ];
-          THINGTOBENCH = ./thingToBench.nix;
-          NIX_PATH="nixpkgs=${prev.path}";
-        } ''
-          bash ${./benchmark.sh} run_all
-        '';
       };
 
       hydraJobs = {
@@ -492,7 +486,7 @@
       });
 
       packages = forAllSystems (system: {
-        inherit (nixpkgsFor.${system}) nix nix-benchmarks;
+        inherit (nixpkgsFor.${system}) nix;
       } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;

misc/zsh/completion.zsh

@@ -1,3 +1,5 @@
+#compdef nix
+
 function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
@@ -18,4 +20,4 @@ function _nix() {
   _describe 'nix' suggestions
 }
 
-compdef _nix nix
+_nix "$@"

src/build-remote/build-remote.cc

@@ -277,7 +277,16 @@ connected:
 
         auto drv = store->readDerivation(*drvPath);
         auto outputHashes = staticOutputHashes(*store, drv);
-        drv.inputSrcs = store->parseStorePathSet(inputs);
+
+        // Hijack the inputs paths of the derivation to include all the paths
+        // that come from the `inputDrvs` set.
+        // We don’t do that for the derivations whose `inputDrvs` is empty
+        // because
+        // 1. It’s not needed
+        // 2. Changing the `inputSrcs` set changes the associated output ids,
+        //  which break CA derivations
+        if (!drv.inputDrvs.empty())
+            drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
 

src/libcmd/installables.cc

@@ -302,14 +302,6 @@ Installable::getCursors(EvalState & state)
     return {{evalCache->getRoot(), ""}};
 }
 
-std::pair<Value*, Pos> Installable::toValue(EvalState & state)
-{
-    auto values = toValues(state);
-    if (values.empty())
-        throw Error("cannot find flake attribute '%s'", what());
-    return {std::get<0>(values[0]), std::get<1>(values[0])};
-}
-
 std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
 Installable::getCursor(EvalState & state)
 {
@@ -386,11 +378,11 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
         state.forceValue(*vRes);
-        return {{vRes, pos, attrPath}};
+        return {vRes, pos};
     }
 
     virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -436,10 +428,12 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
     callFlake(state, lockedFlake, *vFlake);
 
-    auto vRes = state.allocValue();
-    auto gotField = state.lazyGetAttrField(*vFlake, {state.symbols.create("outputs")}, noPos, *vRes);
-    assert(gotField != EvalState::LazyValueType::Missing);
-    return vRes;
+    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    assert(aOutputs);
+
+    state.forceValue(*aOutputs->value);
+
+    return aOutputs->value;
 }
 
 ref<eval_cache::EvalCache> openEvalCache(
@@ -506,34 +500,25 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
-        auto emptyArgs = state->allocBindings(0);
-        try {
-            auto [drvValue, pos] = findAlongAttrPath(
-                *state,
-                attrPath,
-                *emptyArgs,
-                *getFlakeOutputs(*state, *lockedFlake)
-            );
-            Value * v = state->allocValue();
-            if (!state->getAttrField(*drvValue, {state->sDrvPath}, pos, *v))
-                break;
-            auto drvPath = state->forceString(*v);
-            if (!state->getAttrField(*drvValue, {state->sOutPath}, pos, *v))
-                break;
-            auto outPath = state->forceString(*v);
-            if (!state->getAttrField(*drvValue, {state->sOutputName}, pos, *v))
-                break;
-            auto outputName = state->forceString(*v);
+        auto attr = root->findAlongAttrPath(
+            parseAttrPath(*state, attrPath),
+            true
+        );
 
-            auto drvInfo = DerivationInfo{
-                state->store->parseStorePath(drvPath),
-                state->store->maybeParseStorePath(outPath),
-                outputName
-            };
+        if (!attr) continue;
 
-            return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
-        } catch (AttrPathNotFound & e) {
-        }
+        if (!attr->isDerivation())
+            throw Error("flake output attribute '%s' is not a derivation", attrPath);
+
+        auto drvPath = attr->forceDerivation();
+
+        auto drvInfo = DerivationInfo{
+            std::move(drvPath),
+            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
+            attr->getAttr(state->sOutputName)->getString()
+        };
+
+        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
     }
 
     throw Error("flake '%s' does not provide attribute %s",
@@ -547,22 +532,25 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
     return res;
 }
 
-std::vector<std::tuple<Value *, Pos, std::string>>
-InstallableFlake::toValues(EvalState & state)
+std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 {
-    auto vOutputs = getFlakeOutputs(state, *getLockedFlake());
-    auto emptyArgs = state.allocBindings(0);
+    auto lockedFlake = getLockedFlake();
 
-    std::vector<std::tuple<Value *, Pos, std::string>> res;
+    auto vOutputs = getFlakeOutputs(state, *lockedFlake);
+
+    auto emptyArgs = state.allocBindings(0);
 
     for (auto & attrPath : getActualAttrPaths()) {
         try {
             auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
-            res.push_back({v, pos, attrPath});
-        } catch (Error &) {}
+            state.forceValue(*v);
+            return {v, pos};
+        } catch (AttrPathNotFound & e) {
+        }
     }
 
-    return res;
+    throw Error("flake '%s' does not provide attribute %s",
+        flakeRef, showAttrPaths(getActualAttrPaths()));
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>

src/libcmd/installables.hh

@@ -41,11 +41,10 @@ struct Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state)
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }
-    std::pair<Value *, Pos> toValue(EvalState & state);
 
     /* Return a value only if this installable is a store path or a
        symlink to it. */
@@ -110,7 +109,7 @@ struct InstallableFlake : InstallableValue
 
     std::vector<DerivationInfo> toDerivations() override;
 
-    std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override;
+    std::pair<Value *, Pos> toValue(EvalState & state) override;
 
     std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
     getCursors(EvalState & state) override;

src/libexpr/attr-path.cc

@@ -58,23 +58,26 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         Value * vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
+        state.forceValue(*v);
 
         /* It should evaluate to either a set or an expression,
            according to what is specified in the attrPath. */
 
         if (!attrIndex) {
 
+            if (v->type() != nAttrs)
+                throw TypeError(
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            auto v2 = state.allocValue();
-            auto gotField = state.lazyGetAttrField(*v, {state.symbols.create(attr)}, pos, *v2) != EvalState::LazyValueType::Missing;
-            if (!gotField)
+            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
+            if (a == v->attrs->end())
                 throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
-            v = v2;
-            /* Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); */
-            /* v = &*a->value; */
-            /* pos = *a->pos; */
+            v = &*a->value;
+            pos = *a->pos;
         }
 
         else {

src/libexpr/attr-set.hh

@@ -2,7 +2,6 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
-#include "value-cache.hh"
 
 #include <algorithm>
 #include <optional>
@@ -37,7 +36,6 @@ class Bindings
 public:
     typedef uint32_t size_t;
     Pos *pos;
-    ValueCache eval_cache;
 
 private:
     size_t size_, capacity_;

src/libexpr/context.cc

@@ -1,21 +0,0 @@
-#include "context.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
-    } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
-}
-
-std::string encodeContext(std::string_view name, std::string_view path)
-{
-    return "!" + std::string(name) + "!" + std::string(path);
-}
-
-}

src/libexpr/context.hh

@@ -1,11 +0,0 @@
-#include "util.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s);
-
-std::string encodeContext(std::string_view name, std::string_view path);
-
-}

src/libexpr/eval-inline.hh

@@ -46,12 +46,6 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     }
     else if (v.isApp())
         callFunction(*v.app.left, *v.app.right, v, noPos);
-    else if (v.isCachedThunk()) {
-        auto evalCache = v.getEvalCache();
-        v.mkThunk(v.cachedThunk.thunk->env, v.cachedThunk.thunk->expr);
-        forceValue(v, pos);
-        v.setEvalCache(evalCache);
-    }
     else if (v.isBlackhole())
         throwEvalError(pos, "infinite recursion encountered");
 }

src/libexpr/eval.cc

@@ -1,5 +1,4 @@
 #include "eval.hh"
-#include "value-cache.hh"
 #include "hash.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -9,7 +8,6 @@
 #include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
-#include "value-cache.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -124,7 +122,6 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         str << "]";
         break;
     case tThunk:
-    case tCachedThunk:
     case tApp:
         str << "<CODE>";
         break;
@@ -196,7 +193,7 @@ string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tThunk: case tCachedThunk: return "a thunk";
+        case tThunk: return "a thunk";
         case tApp: return "a function application";
         case tBlackhole: return "a black hole";
     default:
@@ -219,7 +216,7 @@ bool Value::isTrivial() const
     return
         internalType != tApp
         && internalType != tPrimOpApp
-        && ((internalType != tThunk && internalType != tCachedThunk)
+        && (internalType != tThunk
             || (dynamic_cast<ExprAttrs *>(thunk.expr)
                 && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
             || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -445,9 +442,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    for (auto [_, cache] : evalCache) {
-        cache->commit();
-    }
 }
 
 
@@ -1113,235 +1107,68 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
     v = *v2;
 }
 
+
+static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
+{
+    std::ostringstream out;
+    bool first = true;
+    for (auto & i : attrPath) {
+        if (!first) out << '.'; else first = false;
+        try {
+            out << getName(i, state, env);
+        } catch (Error & e) {
+            assert(!i.symbol.set());
+            out << "\"${" << *i.expr << "}\"";
+        }
+    }
+    return out.str();
+}
+
+
 unsigned long nrLookups = 0;
 
-std::pair<ValueCache::CacheResult, ValueCache> ValueCache::getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest)
-{
-    if (!rawCache)
-        return { {NoCacheKey}, ValueCache(nullptr) };
-    auto resultingCursor = rawCache->findAlongAttrPath(selector);
-    if (!resultingCursor)
-        return { {CacheMiss}, ValueCache(nullptr) };
-
-    auto cachedValue = resultingCursor->getCachedValue();
-    auto cacheResult = std::visit(
-        overloaded{
-            [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
-            [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
-            [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
-            [&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
-            [&](tree_cache::missing_t x) {
-                return ValueCache::CacheResult{
-                    .returnCode = CacheHit,
-                    .lastQueriedSymbolIfMissing = x.attrName
-                };
-            },
-            [&](tree_cache::string_t s) {
-                PathSet context;
-                for (auto& [pathName, outputName] : s.second) {
-                    // If the cached value depends on some non-existent
-                    // path, we need to discard it and force the evaluation
-                    // to bring back the context in the store
-                    if (!state.store->isValidPath(
-                            state.store->parseStorePath(pathName)))
-                        return ValueCache::CacheResult{UnCacheable};
-                    context.insert("!" + outputName + "!" + pathName);
-                }
-                mkString(dest, s.first, context);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<bool> b) {
-                dest.mkBool(b.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<int64_t> i) {
-                dest.mkInt(i.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<double> d) {
-                dest.mkFloat(d.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-        },
-        cachedValue);
-
-    return { cacheResult, ValueCache(resultingCursor) };
-}
-
-void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
-{
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            nrCacheHits++;
-            break;
-        case ValueCache::CacheMiss:
-            nrCacheMisses++;
-            break;
-        case ValueCache::UnCacheable:
-            nrUncacheable++;
-            break;
-        case ValueCache::NoCacheKey:
-            nrUncached++;
-            break;
-        case ValueCache::Forward:
-            nrCacheHits++;
-            break;
-    };
-}
-
-EvalState::LazyValueType EvalState::lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    auto eval_cache = attrs.getEvalCache();
-    if (eval_cache.isEmpty()) {
-        forceValue(attrs, pos);
-        eval_cache = attrs.getEvalCache();
-    }
-    auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
-    updateCacheStats(cacheResult);
-
-    auto delayValue = [&](ValueCache & cursor) {
-            auto recordAsVar = new ExprCastedVar(&attrs);
-            auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
-            auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
-            thunk->expr = accessExpr;
-            thunk->env = &baseEnv;
-
-            dest.mkCachedThunk(
-                thunk,
-                new ValueCache(cursor)
-            );
-    };
-
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            if (cacheResult.lastQueriedSymbolIfMissing)
-                return LazyValueType::Missing;
-            return LazyValueType::PlainValue;
-        case ValueCache::UnCacheable:
-            delayValue(resultingCursor);
-            return LazyValueType::DelayedUncacheable;
-        case ValueCache::Forward:
-            delayValue(resultingCursor);
-            return LazyValueType::DelayedAttr;
-        default:
-            if(getAttrField(attrs, selector, pos, dest))
-                return LazyValueType::PlainValue;
-            else
-                return LazyValueType::Missing;
-    }
-
-}
-
-bool EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    Pos * pos2 = 0;
-
-    auto eval_cache = attrs.getEvalCache();
-    if (eval_cache.isEmpty()) {
-        forceValue(attrs, pos);
-        eval_cache = attrs.getEvalCache();
-    }
-    auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
-    updateCacheStats(cacheResult);
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            if (cacheResult.lastQueriedSymbolIfMissing)
-                return false;
-            return true;
-        case ValueCache::CacheMiss:
-            resultingCursor = eval_cache;
-            break;
-        case ValueCache::Forward: // FIXME: Handle properly
-        case ValueCache::NoCacheKey:
-        case ValueCache::UnCacheable:
-            ;
-    }
-
-    forceValue(attrs, pos);
-
-    Value * vAttrs = &attrs;
-    try {
-        for (auto & name : selector) {
-            nrLookups++;
-            Bindings::iterator j;
-            if (vAttrs->type() != nAttrs ||
-                (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                return false;
-            }
-            vAttrs = j->value;
-            pos2 = j->pos;
-            try {
-                forceValue(*vAttrs, pos2 != NULL ? *pos2 : pos );
-            } catch (EvalError & e) {
-                resultingCursor.addFailedChild(name, e);
-                throw;
-            }
-            if (cacheResult.returnCode == ValueCache::CacheMiss) {
-                resultingCursor = resultingCursor.addChild(name, *vAttrs);
-                vAttrs->setEvalCache(resultingCursor);
-            }
-            if (countCalls && pos2) attrSelects[*pos2]++;
-        }
-
-    } catch (Error & e) {
-        if (pos2 && pos2->file != sDerivationNix) {
-            vector<string> strSelector;
-            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", concatStringsSep(".", selector));
-        }
-        throw;
-    }
-
-    if (cacheResult.returnCode == ValueCache::Forward)
-        vAttrs->setEvalCache(resultingCursor);
-
-    dest = *vAttrs;
-    return true;
-}
-
-void EvalState::getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    if (!getAttrField(attrs, selector, pos, dest))
-        throw Error("Missing attribute path '%s'", "ImTooLazyToImplementThisRightNow");
-}
-
-std::vector<Symbol> EvalState::getFields(Value & attrs, const Pos & pos)
-{
-    auto eval_cache = attrs.getEvalCache();
-    if (eval_cache.isEmpty()) {
-        forceValue(attrs, pos);
-        eval_cache = attrs.getEvalCache();
-    }
-    if (auto attrNames = eval_cache.listChildren(symbols)) {
-        return *attrNames;
-    }
-
-    forceAttrs(attrs);
-    std::vector<Symbol> res;
-    for (auto & attr : *attrs.attrs)
-        res.push_back(attr.name);
-    return res;
-}
-
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
+    Pos * pos2 = 0;
+    Value * vAttrs = &vTmp;
 
     e->eval(state, env, vTmp);
 
-    std::vector<Symbol> selector;
-    for (auto & i : attrPath)
-        selector.push_back(getName(i, state, env));
+    try {
 
-    bool gotField = state.getAttrField(vTmp, selector, pos, v);
-    if (!gotField) {
-        if (def) {
-            def->eval(state, env, v);
-            return;
-        } else {
-            throwEvalError(pos, "Missing field");
+        for (auto & i : attrPath) {
+            nrLookups++;
+            Bindings::iterator j;
+            Symbol name = getName(i, state, env);
+            if (def) {
+                state.forceValue(*vAttrs, pos);
+                if (vAttrs->type() != nAttrs ||
+                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                {
+                    def->eval(state, env, v);
+                    return;
+                }
+            } else {
+                state.forceAttrs(*vAttrs, pos);
+                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                    throwEvalError(pos, "attribute '%1%' missing", name);
+            }
+            vAttrs = j->value;
+            pos2 = j->pos;
+            if (state.countCalls && pos2) state.attrSelects[*pos2]++;
         }
+
+        state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
+
+    } catch (Error & e) {
+        if (pos2 && pos2->file != state.sDerivationNix)
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
+        throw;
     }
 
+    v = *vAttrs;
 }
 
 
@@ -1524,38 +1351,9 @@ void EvalState::incrFunctionCall(ExprLambda * fun)
     functionCalls[fun]++;
 }
 
-std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
-{
-    if (!rawCache)
-        return std::nullopt;
-    return rawCache->getCachedValue();
-}
-
-std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
-{
-    auto ret = std::vector<Symbol>();
-    if (rawCache) {
-        auto cachedValue = rawCache->getCachedValue();
-            if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
-                for (auto & fieldStr : rawCache->getChildren())
-                    ret.push_back(symbols.create(fieldStr));
-            }
-            return ret;
-    }
-    return std::nullopt;
-}
 
 void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
-    if (auto evalCache = fun.getEvalCache(); !evalCache.isEmpty()) {
-        if (auto cacheValue = evalCache.getRawValue()) {
-            if (std::holds_alternative<tree_cache::attributeSet_t>(*cacheValue)) {
-                res = fun;
-                return;
-            }
-        }
-    }
-
     forceValue(fun);
 
     if (fun.type() == nAttrs) {
@@ -1901,6 +1699,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -1919,102 +1729,6 @@ std::vector<std::pair<Path, std::string>> Value::getContext()
     return res;
 }
 
-ValueCache & Value::getEvalCache()
-{
-    if (internalType == tAttrs) {
-        return attrs->eval_cache;
-    } else if (internalType == tCachedThunk) {
-        return *cachedThunk.cache;
-    } else {
-        return ValueCache::empty;
-    }
-}
-
-ValueCache ValueCache::empty = ValueCache(nullptr);
-
-tree_cache::AttrValue cachedValueFor(Value& v)
-{
-    tree_cache::AttrValue valueToCache;
-    switch (v.type()) {
-        case nThunk:
-            valueToCache = tree_cache::thunk_t{};
-            break;
-        case nNull:
-        case nList:
-        case nFunction:
-        case nExternal:
-            valueToCache = tree_cache::unknown_t{};
-            break;
-        case nBool:
-            valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
-            break;
-        case nString:
-            valueToCache = tree_cache::string_t{
-                v.string.s,
-                v.getContext()
-            };
-            break;
-        case nPath:
-            valueToCache = tree_cache::string_t{
-                v.path,
-                {}
-            };
-            break;
-        case nAttrs:
-            valueToCache = tree_cache::attributeSet_t{};
-            break;
-        case nInt:
-            valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
-            break;
-        case nFloat:
-            valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
-            break;
-    };
-    return valueToCache;
-}
-
-ValueCache ValueCache::addChild(const Symbol& name, Value& value)
-{
-    if (!rawCache)
-        return ValueCache::empty;
-
-    auto cachedValue = cachedValueFor(value);
-    auto ret = ValueCache(rawCache->addChild(name, cachedValue));
-    if (value.type() == nAttrs)
-        ret.addAttrSetChilds(*value.attrs);
-    return ret;
-}
-
-
-ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
-{
-    if (!rawCache) return ValueCache::empty;
-    return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
-}
-
-void ValueCache::addAttrSetChilds(Bindings & children)
-{
-    if (!rawCache) return;
-    for (auto & attr : children) {
-        // We could in theory directly store the value, but that would cause
-        // an infinite recursion in case of a cyclic attrset.
-        // So in a first time, just store a thunk
-        if (attr.value->type() == nAttrs)
-            rawCache->addChild(attr.name, tree_cache::thunk_t{});
-        else
-            addChild(attr.name, *attr.value);
-    }
-
-}
-
-void Value::setEvalCache(ValueCache & newCache)
-{
-    if (internalType == tAttrs) {
-        attrs->eval_cache = newCache;
-    } else if (internalType == tCachedThunk) {
-        cachedThunk.cache = &newCache;
-    }
-}
 
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
@@ -2297,13 +2011,6 @@ void EvalState::printStats()
         topObj.attr("nrLookups", nrLookups);
         topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
         topObj.attr("nrFunctionCalls", nrFunctionCalls);
-        {
-            auto cache = topObj.object("evalCache");
-            cache.attr("nrCacheMisses", nrCacheMisses);
-            cache.attr("nrCacheHits", nrCacheHits);
-            cache.attr("nrUncached", nrUncached);
-            cache.attr("nrUncacheable", nrUncacheable);
-        }
 #if HAVE_BOEHMGC
         {
             auto gc = topObj.object("gc");
@@ -2404,21 +2111,6 @@ Strings EvalSettings::getDefaultNixPath()
     return res;
 }
 
-std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
-{
-    if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
-        return iter->second;
-
-    if (!(evalSettings.useEvalCache && evalSettings.pureEval))
-        return nullptr;
-    auto thisCache = tree_cache::Cache::tryCreate(
-            cacheKey,
-            symbols
-    );
-    evalCache.insert({cacheKey, thisCache});
-    return thisCache;
-}
-
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);

src/libexpr/eval.hh

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "attr-set.hh"
-#include "context.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
@@ -116,7 +115,6 @@ private:
     typedef std::map<Path, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
-    std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
 
     SearchPath searchPath;
 
@@ -133,8 +131,6 @@ public:
     EvalState(const Strings & _searchPath, ref<Store> store);
     ~EvalState();
 
-    std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
-
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
@@ -314,8 +310,6 @@ public:
 
     void realiseContext(const PathSet & context);
 
-    void updateCacheStats(ValueCache::CacheResult);
-
 private:
 
     unsigned long nrEnvs = 0;
@@ -329,10 +323,6 @@ private:
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
-    unsigned long nrCacheHits = 0;
-    unsigned long nrCacheMisses = 0;
-    unsigned long nrUncacheable = 0;
-    unsigned long nrUncached = 0;
 
     bool countCalls;
 
@@ -352,27 +342,6 @@ private:
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
-public:
-
-    bool getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-
-    enum struct LazyValueType {
-        PlainValue,
-        DelayedUncacheable,
-        DelayedAttr,
-        Missing,
-    };
-
-    // Similar to `getAttrField`, but if the cache says that the result is an
-    // attribute set, just return a thunk to it rather than forcing it.
-    LazyValueType lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-
-    // Similar to `getAttrField`, but throws an `Error` if the field can’t be
-    // found
-    void getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-
-    std::vector<Symbol> getFields(Value & attrs, const Pos & pos);
 };
 
 
@@ -380,6 +349,10 @@ public:
 string showType(ValueType type);
 string showType(const Value & v);
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s);
+
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
 

src/libexpr/flake/flake.cc

@@ -596,44 +596,30 @@ void callFlake(EvalState & state,
     const LockedFlake & lockedFlake,
     Value & vRes)
 {
-    auto lockExpr = new ExprString(
-        state.symbols.create(lockedFlake.lockFile.to_string())
-    );
-    auto subdirExpr = new ExprString(
-        state.symbols.create(lockedFlake.flake.lockedRef.subdir)
-    );
-
+    auto vLocks = state.allocValue();
     auto vRootSrc = state.allocValue();
+    auto vRootSubdir = state.allocValue();
+    auto vTmp1 = state.allocValue();
+    auto vTmp2 = state.allocValue();
+
+    mkString(*vLocks, lockedFlake.lockFile.to_string());
+
     emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
 
-    static Expr * callFlakeExpr = nullptr;
-    if (!callFlakeExpr) {
-        callFlakeExpr = state.parseExprFromString(
+    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+
+    static RootValue vCallFlake = nullptr;
+
+    if (!vCallFlake) {
+        vCallFlake = allocRootValue(state.allocValue());
+        state.eval(state.parseExprFromString(
             #include "call-flake.nix.gen.hh"
-            , "/");
+            , "/"), **vCallFlake);
     }
 
-    auto resExpr = new ExprApp(
-        new ExprApp(
-            new ExprApp(
-                callFlakeExpr,
-                lockExpr
-            ),
-            new ExprCastedVar(vRootSrc)
-        ),
-        subdirExpr
-    );
-    auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
-    thunk->expr = resExpr;
-    thunk->env = &state.baseEnv;
-    auto fingerprint = lockedFlake.getFingerprint();
-    auto treeCache = state.openTreeCache(fingerprint);
-    auto cacheRoot = treeCache ? treeCache->getRoot() : nullptr;
-    auto evalCache = new ValueCache(cacheRoot);
-    vRes.mkCachedThunk(
-        thunk,
-        evalCache
-    );
+    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)

src/libexpr/nixexpr.cc

@@ -440,18 +440,6 @@ string ExprLambda::showNamePos() const
     return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
 }
 
-void ExprCastedVar::show(std::ostream & str) const {
-    std::set<const Value*> active;
-    printValue(str, active, *v);
-}
-
-void ExprCastedVar::bindVars(const StaticEnv & env) {}
-void ExprCastedVar::eval(EvalState & state, Env & env, Value & v) {
-    v = std::move(*this->v);
-}
-Value * ExprCastedVar::maybeThunk(EvalState & state, Env & env) {
-    return v;
-}
 
 
 /* Symbol table. */

src/libexpr/nixexpr.hh

@@ -165,12 +165,7 @@ struct ExprSelect : Expr
     Expr * e, * def;
     AttrPath attrPath;
     ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
-    ExprSelect(const Pos & pos, Expr * e, const Symbol & name)
-        : ExprSelect(pos, e, std::vector{name}) {};
-    ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & symbolicAttrPath) : pos(pos), e(e), def(0) {
-        for (auto & name : symbolicAttrPath)
-            attrPath.push_back(AttrName(name));
-    };
+    ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     COMMON_METHODS
 };
 
@@ -339,14 +334,6 @@ struct ExprPos : Expr
     COMMON_METHODS
 };
 
-struct ExprCastedVar : Expr
-{
-    Value * v;
-    ExprCastedVar(Value * v) : v(v) {};
-    Value * maybeThunk(EvalState & state, Env & env);
-    COMMON_METHODS
-};
-
 
 /* Static environments are used to map variable names onto (level,
    displacement) pairs used to obtain the value of the variable at

src/libexpr/tree-cache.cc

@@ -1,392 +0,0 @@
-#include "tree-cache.hh"
-#include "sqlite.hh"
-#include "store-api.hh"
-#include "context.hh"
-
-namespace nix::tree_cache {
-
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    id          integer primary key autoincrement not null,
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    context     text,
-    unique      (parent, name)
-);
-
-create index if not exists IndexByParent on Attributes(parent, name);
-)sql";
-
-struct AttrDb
-{
-    std::atomic_bool failed{false};
-
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt updateAttribute;
-        SQLiteStmt insertAttributeWithContext;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->updateAttribute.create(state->db,
-            "update Attributes set type = ?, value = ?, context = ? where id = ?");
-
-        state->insertAttributeWithContext.create(state->db,
-            "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select id, type, value, context from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name, type from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            if (!failed)
-                state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    template<typename F>
-    AttrId doSQLite(F && fun)
-    {
-        if (failed) return 0;
-        try {
-            return fun();
-        } catch (SQLiteError &) {
-            ignoreException();
-            failed = true;
-            return 0;
-        }
-    }
-
-    /**
-     * Store a leaf of the tree in the db
-     */
-    AttrId addEntry(
-        const AttrKey & key,
-        const AttrValue & value)
-    {
-        return doSQLite([&]()
-        {
-            auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->insertAttributeWithContext.use()
-                (key.first)
-                (key.second)
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                .exec();
-            AttrId rowId = state->db.getLastInsertedRowId();
-            assert(rowId);
-            return rowId;
-        });
-    }
-
-    std::optional<AttrId> getId(const AttrKey& key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return std::nullopt;
-
-        return (AttrType) queryAttribute.getInt(0);
-    }
-
-    AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
-    {
-        debug("cache: miss for the attribute %s", key.second);
-        if (auto existingId = getId(key)) {
-            setValue(*existingId, value);
-            return *existingId;
-        }
-        return addEntry(key, value);
-    }
-
-    void setValue(const AttrId & id, const AttrValue & value)
-    {
-        auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->updateAttribute.use()
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                (id)
-                .exec();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        switch (type) {
-            case AttrType::Attrs: {
-                return {{rowId, attributeSet_t()}};
-            }
-            case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
-                if (!queryAttribute.isNull(3))
-                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
-                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
-            }
-            case AttrType::Bool:
-                return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
-            case AttrType::Int:
-                return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
-            case AttrType::Double:
-                return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
-            case AttrType::Unknown:
-                return {{rowId, unknown_t{}}};
-            case AttrType::Thunk:
-                return {{rowId, thunk_t{}}};
-            case AttrType::Missing:
-                return {{rowId, missing_t{key.second}}};
-            case AttrType::Failed:
-                return {{rowId, failed_t{queryAttribute.getStr(2)}}};
-            default:
-                throw Error("unexpected type in evaluation cache");
-        }
-    }
-
-    std::vector<std::string> getChildren(AttrId parentId)
-    {
-        std::vector<std::string> res;
-        auto state(_state->lock());
-
-        auto queryAttributes(state->queryAttributes.use()(parentId));
-
-        while (queryAttributes.next()) {
-            if (queryAttributes.getInt(1) != AttrType::Missing)
-            res.push_back(queryAttributes.getStr(0));
-        }
-
-        return res;
-    }
-};
-
-Cache::Cache(const Hash & useCache,
-        SymbolTable & symbols)
-    : db(std::make_shared<AttrDb>(useCache))
-    , symbols(symbols)
-    , rootSymbol(symbols.create("%root%"))
-{
-}
-
-std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
-{
-    try {
-        return std::make_shared<Cache>(useCache, symbols);
-    } catch (SQLiteError &) {
-        ignoreException();
-        return nullptr;
-    }
-}
-
-void Cache::commit()
-{
-    if (db) {
-        debug("Saving the cache");
-        auto state(db->_state->lock());
-        if (state->txn->active) {
-            state->txn->commit();
-            state->txn.reset();
-            state->txn = std::make_unique<SQLiteTxn>(state->db);
-        }
-    }
-}
-
-Cursor::Ref Cache::getRoot()
-{
-    return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrValue& value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({root->db->setOrUpdate(getKey(), value), value})
-{
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrId & id,
-    const AttrValue & value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({id, value})
-{
-}
-
-
-AttrKey Cursor::getKey()
-{
-    if (!parentId)
-        return {0, root->rootSymbol};
-    return {*parentId, label};
-}
-
-AttrValue Cursor::getCachedValue()
-{
-    return cachedValue.second;
-}
-
-void Cursor::setValue(const AttrValue & v)
-{
-    root->db->setValue(cachedValue.first, v);
-    cachedValue.second = v;
-}
-
-Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
-{
-    Parent parent = {{*this, attrPath}};
-    auto childCursor = new Cursor(
-        root,
-        parent,
-        v
-    );
-    return childCursor;
-}
-
-std::vector<std::string> Cursor::getChildren()
-{
-    return root->db->getChildren(cachedValue.first);
-}
-
-std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
-{
-    auto cursorAtPath = findAlongAttrPath(attrPath);
-    if (cursorAtPath)
-        return cursorAtPath->getChildren();
-    return std::nullopt;
-}
-
-Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
-{
-    auto rawAttr = root->db->getValue({cachedValue.first, name});
-    if (rawAttr) {
-        Parent parent = {{*this, name}};
-        debug("cache: hit for the attribute %s", cachedValue.first);
-        return new Cursor (
-            root, parent, rawAttr->first,
-            rawAttr->second);
-    }
-    if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
-        // If the parent is an attribute set but we're not present in the db,
-        // then we're not a member of this attribute set. So mark as missing
-        return addChild(name, missing_t{name});
-    }
-    return nullptr;
-}
-
-Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
-{
-    auto currentCursor = this;
-    for (auto & currentAccessor : attrPath) {
-        currentCursor = currentCursor->maybeGetAttr(currentAccessor);
-        if (!currentCursor)
-            break;
-        if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
-            break;
-        if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
-            break;
-    }
-    return currentCursor;
-}
-
-const RawValue RawValue::fromVariant(const AttrValue & value)
-{
-    RawValue res;
-    std::visit(overloaded{
-      [&](attributeSet_t x) { res.type = AttrType::Attrs; },
-      [&](string_t x) {
-        res.type = AttrType::String;
-        res.value = x.first;
-        res.context = x.second;
-      },
-      [&](wrapped_basetype<bool> x) {
-        res.type = AttrType::Bool;
-        res.value = x.value ? "1" : "0";
-      },
-      [&](wrapped_basetype<int64_t> x) {
-        res.type = AttrType::Int;
-        res.value = std::to_string(x.value);
-      },
-      [&](wrapped_basetype<double> x) {
-        res.type = AttrType::Double;
-        res.value = std::to_string(x.value);
-      },
-      [&](unknown_t x) { res.type = AttrType::Unknown; },
-      [&](missing_t x) { res.type = AttrType::Missing; },
-      [&](thunk_t x) { res.type = AttrType::Thunk; },
-      [&](failed_t x) {
-          res.type = AttrType::Failed;
-          res.value = x.error;
-      }
-    }, value);
-    return res;
-}
-
-std::string RawValue::serializeContext() const
-{
-    std::string res;
-    for (auto & elt : context) {
-        res.append(encodeContext(elt.second, elt.first));
-        res.push_back(' ');
-    }
-    if (!res.empty())
-        res.pop_back(); // Remove the trailing space
-    return res;
-}
-
-}

src/libexpr/tree-cache.hh

@@ -1,156 +0,0 @@
-/**
- * caching for a tree-like data structure (like Nix values)
- *
- * The cache is an sqlite db whose rows are the nodes of the tree, with a
- * pointer to their parent (except for the root of course)
- */
-
-#pragma once
-
-#include "sync.hh"
-#include "hash.hh"
-#include "symbol-table.hh"
-
-#include <functional>
-#include <variant>
-
-namespace nix::tree_cache {
-
-struct AttrDb;
-class Cursor;
-
-class Cache : public std::enable_shared_from_this<Cache>
-{
-private:
-    friend class Cursor;
-
-    /**
-     * The database holding the cache
-     */
-    std::shared_ptr<AttrDb> db;
-
-    SymbolTable & symbols;
-
-    /**
-     * Distinguished symbol indicating the root of the tree
-     */
-    const Symbol rootSymbol;
-
-public:
-
-    Cache(
-        const Hash & useCache,
-        SymbolTable & symbols
-    );
-
-    static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
-
-    Cursor * getRoot();
-
-    /**
-     * Flush the cache to disk
-     */
-    void commit();
-};
-
-enum AttrType {
-    Unknown = 0,
-    Attrs = 1,
-    String = 2,
-    Bool = 3,
-    Int = 4,
-    Double = 5,
-    Thunk = 6,
-    Missing = 7, // Missing fields of attribute sets
-    Failed = 8,
-};
-
-struct attributeSet_t {};
-struct unknown_t {};
-struct thunk_t {};
-struct failed_t { string error; };
-struct missing_t { Symbol attrName; };
-
-// Putting several different primitive types in an `std::variant` partially
-// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
-// from one to another which breaks the exhaustiveness check.
-// So we wrap them in a trivial class just to force the disambiguation
-template<typename T>
-struct wrapped_basetype{ T value; };
-
-typedef uint64_t AttrId;
-
-typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
-
-typedef std::variant<
-    attributeSet_t,
-    string_t,
-    unknown_t,
-    thunk_t,
-    missing_t,
-    failed_t,
-    wrapped_basetype<bool>,
-    wrapped_basetype<int64_t>,
-    wrapped_basetype<double>
-> AttrValue;
-
-struct RawValue {
-    AttrType type;
-    std::optional<std::string> value;
-    std::vector<std::pair<Path, std::string>> context;
-
-    std::string serializeContext() const;
-
-    static const RawValue fromVariant(const AttrValue&);
-    AttrValue toVariant() const;
-};
-
-/**
- * View inside the cache.
- *
- * A `Cursor` represents a node in the cached tree (be it a leaf or not)
- */
-class Cursor : public std::enable_shared_from_this<Cursor>
-{
-    /**
-     * The overall cache of which this cursor is a view
-     */
-    ref<Cache> root;
-
-    typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
-
-    std::optional<AttrId> parentId;
-    Symbol label;
-
-    std::pair<AttrId, AttrValue> cachedValue;
-
-    /**
-     * Get the identifier for this node in the database
-     */
-    AttrKey getKey();
-
-public:
-
-    using Ref = Cursor*;
-
-    // Create a new cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
-    // Build a cursor from an existing cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
-
-    AttrValue getCachedValue();
-
-    void setValue(const AttrValue & v);
-
-    Ref addChild(const Symbol & attrPath, const AttrValue & v);
-
-    Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
-    Ref maybeGetAttr(const Symbol & attrPath);
-
-
-    std::vector<std::string> getChildren();
-    std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
-};
-
-}

src/libexpr/value-cache.hh

@@ -1,51 +0,0 @@
-#pragma once
-#include "tree-cache.hh"
-
-namespace nix {
-struct Value;
-class EvalState;
-class Bindings;
-
-class ValueCache {
-    tree_cache::Cursor::Ref rawCache;
-
-public:
-
-    ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
-
-    static ValueCache empty;
-
-    bool isEmpty () { return rawCache == nullptr; }
-
-    enum ReturnCode {
-        // The cache result was an attribute set, so we forward it later in the
-        // chain
-        Forward,
-        CacheMiss,
-        CacheHit,
-        UnCacheable,
-        NoCacheKey,
-    };
-
-    struct CacheResult {
-        ReturnCode returnCode;
-
-        // In case the query returns a `missing_t`, the symbol that's missing
-        std::optional<Symbol> lastQueriedSymbolIfMissing;
-    };
-    std::pair<CacheResult, ValueCache> getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest);
-
-    ValueCache addChild(const Symbol & attrName, Value & value);
-    ValueCache addFailedChild(const Symbol & attrName, const Error & error);
-    ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
-    void addAttrSetChilds(Bindings & children);
-    void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
-
-    std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
-    std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
-
-    std::optional<tree_cache::AttrValue> getRawValue();
-
-    ValueCache() : rawCache(nullptr) {}
-};
-}

src/libexpr/value.hh

@@ -23,7 +23,6 @@ typedef enum {
     tApp,
     tLambda,
     tBlackhole,
-    tCachedThunk,
     tPrimOp,
     tPrimOpApp,
     tExternal,
@@ -57,7 +56,6 @@ struct Pos;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
-class ValueCache;
 
 
 typedef int64_t NixInt;
@@ -105,10 +103,6 @@ class ExternalValueBase
 
 std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 
-struct Thunk {
-    Env * env;
-    Expr * expr;
-};
 
 struct Value
 {
@@ -128,7 +122,6 @@ public:
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
     inline bool isBlackhole() const { return internalType == tBlackhole; };
-    inline bool isCachedThunk() const { return internalType == tCachedThunk; };
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
@@ -172,13 +165,10 @@ public:
             Value * * elems;
         } bigList;
         Value * smallList[2];
-        Thunk thunk;
         struct {
-            Thunk * thunk;
-            // This is a pointer only to prevent a recursive import as
-            // `EvalCache` is already a pointer so would fit very nicely here.
-            ValueCache * cache;
-        } cachedThunk;
+            Env * env;
+            Expr * expr;
+        } thunk;
         struct {
             Value * left, * right;
         } app;
@@ -209,7 +199,7 @@ public:
             case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
-            case tThunk: case tApp: case tBlackhole: case tCachedThunk: return nThunk;
+            case tThunk: case tApp: case tBlackhole: return nThunk;
         }
         abort();
     }
@@ -282,13 +272,6 @@ public:
         thunk.expr = ex;
     }
 
-    inline void mkCachedThunk(Thunk * t, ValueCache * cache)
-    {
-        internalType = tCachedThunk;
-        cachedThunk.thunk = t;
-        cachedThunk.cache = cache;
-    }
-
     inline void mkApp(Value * l, Value * r)
     {
         internalType = tApp;
@@ -366,9 +349,6 @@ public:
     bool isTrivial() const;
 
     std::vector<std::pair<Path, std::string>> getContext();
-
-    ValueCache & getEvalCache();
-    void setEvalCache(ValueCache &);
 };
 
 

src/libstore/build/derivation-goal.cc

@@ -739,6 +739,63 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 {
 }
 
+void runPostBuildHook(
+    Store & store,
+    Logger & logger,
+    const StorePath & drvPath,
+    StorePathSet outputPaths
+)
+{
+    auto hook = settings.postBuildHook;
+    if (hook == "")
+        return;
+
+    Activity act(logger, lvlInfo, actPostBuildHook,
+            fmt("running post-build-hook '%s'", settings.postBuildHook),
+            Logger::Fields{store.printStorePath(drvPath)});
+    PushActivity pact(act.id);
+    std::map<std::string, std::string> hookEnvironment = getEnv();
+
+    hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
+    hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
+
+    RunOptions opts(settings.postBuildHook, {});
+    opts.environment = hookEnvironment;
+
+    struct LogSink : Sink {
+        Activity & act;
+        std::string currentLine;
+
+        LogSink(Activity & act) : act(act) { }
+
+        void operator() (std::string_view data) override {
+            for (auto c : data) {
+                if (c == '\n') {
+                    flushLine();
+                } else {
+                    currentLine += c;
+                }
+            }
+        }
+
+        void flushLine() {
+            act.result(resPostBuildLogLine, currentLine);
+            currentLine.clear();
+        }
+
+        ~LogSink() {
+            if (currentLine != "") {
+                currentLine += '\n';
+                flushLine();
+            }
+        }
+    };
+    LogSink sink(act);
+
+    opts.standardOut = &sink;
+    opts.mergeStderrToStdout = true;
+    runProgram2(opts);
+}
 
 void DerivationGoal::buildDone()
 {
@@ -804,57 +861,15 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (std::string_view data) override {
-                    for (auto c : data) {
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
+        StorePathSet outputPaths;
+        for (auto & [_, path] : finalOutputs)
+            outputPaths.insert(path);
+        runPostBuildHook(
+            worker.store,
+            *logger,
+            drvPath,
+            outputPaths
+        );
 
         if (buildMode == bmCheck) {
             cleanupPostOutputsRegisteredModeCheck();
@@ -910,6 +925,8 @@ void DerivationGoal::resolvedFinished() {
 
     auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
 
+    StorePathSet outputPaths;
+
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
@@ -927,8 +944,10 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
+            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
+            outputPaths.insert(realisation->outPath);
         } else {
             // If we don't have a realisation, then it must mean that something
             // failed when building the resolved drv
@@ -936,6 +955,13 @@ void DerivationGoal::resolvedFinished() {
         }
     }
 
+    runPostBuildHook(
+        worker.store,
+        *logger,
+        drvPath,
+        outputPaths
+    );
+
     // This is potentially a bit fishy in terms of error reporting. Not sure
     // how to do it in a cleaner way
     amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);

src/libstore/build/drv-output-substitution-goal.cc

@@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
 void DrvOutputSubstitutionGoal::init()
 {
     trace("init");
+
+    /* If the derivation already exists, we’re done */
+    if (worker.store.queryRealisation(id)) {
+        amDone(ecSuccess);
+        return;
+    }
+
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
     tryNext();
 }
@@ -53,6 +60,26 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
+    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+        if (depId != id) {
+            if (auto localOutputInfo = worker.store.queryRealisation(depId);
+                localOutputInfo && localOutputInfo->outPath != depPath) {
+                warn(
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                    "Local:  %s\n"
+                    "Remote: %s",
+                    sub->getUri(),
+                    depId.to_string(),
+                    worker.store.printStorePath(localOutputInfo->outPath),
+                    worker.store.printStorePath(depPath)
+                );
+                tryNext();
+                return;
+            }
+            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+        }
+    }
+
     addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
     if (waitees.empty()) outPathValid();

src/libstore/build/local-derivation-goal.cc

@@ -292,7 +292,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         auto & localStore = getLocalStore();
         uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
         struct statvfs st;
-        if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
+        if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
             (uint64_t) st.f_bavail * st.f_bsize < required)
             diskFull = true;
         if (statvfs(tmpDir.c_str(), &st) == 0 &&
@@ -417,7 +417,7 @@ void LocalDerivationGoal::startBuilder()
     }
 
     auto & localStore = getLocalStore();
-    if (localStore.storeDir != localStore.realStoreDir) {
+    if (localStore.storeDir != localStore.realStoreDir.get()) {
         #if __linux__
             useChroot = true;
         #else
@@ -726,7 +726,7 @@ void LocalDerivationGoal::startBuilder()
     Path logFile = openLogFile();
 
     /* Create a pipe to get the output of the builder. */
-    //builderOut.create();
+    builderOut.create();
 
     builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
     if (!builderOut.readSide)
@@ -1333,13 +1333,18 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
     // XXX: This should probably be allowed if the realisation corresponds to
     // an allowed derivation
-    { throw Error("queryRealisation"); }
+    {
+        if (!goal.isAllowed(id))
+            throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
+        return next->queryRealisation(id);
+    }
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
         StorePathSet newPaths;
+        std::set<Realisation> newRealisations;
 
         for (auto & req : paths) {
             if (!goal.isAllowed(req))
@@ -1352,16 +1357,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             auto p =  std::get_if<DerivedPath::Built>(&path);
             if (!p) continue;
             auto & bfd = *p;
+            auto drv = readDerivation(bfd.drvPath);
+            auto drvHashes = staticOutputHashes(*this, drv);
             auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
             for (auto & [outputName, outputPath] : outputs)
-                if (wantOutput(outputName, bfd.outputs))
+                if (wantOutput(outputName, bfd.outputs)) {
                     newPaths.insert(outputPath);
+                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                        auto thisRealisation = next->queryRealisation(
+                            DrvOutput{drvHashes.at(outputName), outputName}
+                        );
+                        assert(thisRealisation);
+                        newRealisations.insert(*thisRealisation);
+                    }
+                }
         }
 
         StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
             goal.addDependency(path);
+        for (auto & real : Realisation::closure(*next, newRealisations))
+            goal.addedDrvOutputs.insert(real.id);
     }
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -2464,6 +2481,7 @@ void LocalDerivationGoal::registerOutputs()
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         PathLocks dynamicOutputLock;
+        dynamicOutputLock.setDeletion(true);
         auto optFixedPath = output.path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)
@@ -2487,6 +2505,7 @@ void LocalDerivationGoal::registerOutputs()
                 assert(newInfo.ca);
             } else {
                 auto destPath = worker.store.toRealPath(finalDestPath);
+                deletePath(destPath);
                 movePath(actualPath, destPath);
                 actualPath = destPath;
             }

src/libstore/build/local-derivation-goal.hh

@@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Paths that were added via recursive Nix calls. */
     StorePathSet addedPaths;
 
+    /* Realisations that were added via recursive Nix calls. */
+    std::set<DrvOutput> addedDrvOutputs;
+
     /* Recursive Nix calls are only allowed to build or realize paths
        in the original input closure or added via a recursive Nix call
        (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
+    bool isAllowed(const DrvOutput & id)
+    {
+        return addedDrvOutputs.count(id);
+    }
+
     bool isAllowed(const DerivedPath & req);
 
     friend struct RestrictedStore;

src/libstore/ca-specific-schema.sql

@@ -3,10 +3,19 @@
 -- is enabled
 
 create table if not exists Realisations (
+    id integer primary key autoincrement not null,
     drvPath text not null,
     outputName text not null, -- symbolic output id, usually "out"
     outputPath integer not null,
     signatures text, -- space-separated list
-    primary key (drvPath, outputName),
     foreign key (outputPath) references ValidPaths(id) on delete cascade
 );
+
+create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+create table if not exists RealisationsRefs (
+    referrer integer not null,
+    realisationReference integer,
+    foreign key (referrer) references Realisations(id) on delete cascade,
+    foreign key (realisationReference) references Realisations(id) on delete restrict
+);

src/libstore/daemon.cc

@@ -885,10 +885,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopRegisterDrvOutput: {
         logger->startWork();
-        auto outputId = DrvOutput::parse(readString(from));
-        auto outputPath = StorePath(readString(from));
-        store->registerDrvOutput(Realisation{
-            .id = outputId, .outPath = outputPath});
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            auto outputId = DrvOutput::parse(readString(from));
+            auto outputPath = StorePath(readString(from));
+            store->registerDrvOutput(Realisation{
+                .id = outputId, .outPath = outputPath});
+        } else {
+            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
+            store->registerDrvOutput(realisation);
+        }
         logger->stopWork();
         break;
     }
@@ -898,9 +903,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto outputId = DrvOutput::parse(readString(from));
         auto info = store->queryRealisation(outputId);
         logger->stopWork();
-        std::set<StorePath> outPaths;
-        if (info) outPaths.insert(info->outPath);
-        worker_proto::write(*store, to, outPaths);
+        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
+            std::set<StorePath> outPaths;
+            if (info) outPaths.insert(info->outPath);
+            worker_proto::write(*store, to, outPaths);
+        } else {
+            std::set<Realisation> realisations;
+            if (info) realisations.insert(*info);
+            worker_proto::write(*store, to, realisations);
+        }
         break;
     }
 

src/libstore/gc.cc

@@ -775,7 +775,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         try {
 
-            AutoCloseDir dir(opendir(realStoreDir.c_str()));
+            AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
             if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
             /* Read the store and immediately delete all paths that
@@ -856,7 +856,7 @@ void LocalStore::autoGC(bool sync)
             return std::stoll(readFile(*fakeFreeSpaceFile));
 
         struct statvfs st;
-        if (statvfs(realStoreDir.c_str(), &st))
+        if (statvfs(realStoreDir.get().c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
 
         return (uint64_t) st.f_bavail * st.f_frsize;

src/libstore/local-fs-store.hh

@@ -18,6 +18,9 @@ struct LocalFSStoreConfig : virtual StoreConfig
     const PathSetting logDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
         "log", "directory where Nix will store state"};
+    const PathSetting realStoreDir{(StoreConfig*) this, false,
+        rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
+        "physical path to the Nix store"};
 };
 
 class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
@@ -34,7 +37,7 @@ public:
     /* Register a permanent GC root. */
     Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
 
-    virtual Path getRealStoreDir() { return storeDir; }
+    virtual Path getRealStoreDir() { return realStoreDir; }
 
     Path toRealPath(const Path & storePath) override
     {

src/libstore/local-store.cc

@@ -53,12 +53,15 @@ struct LocalStore::State::Stmts {
     SQLiteStmt InvalidatePath;
     SQLiteStmt AddDerivationOutput;
     SQLiteStmt RegisterRealisedOutput;
+    SQLiteStmt UpdateRealisedOutput;
     SQLiteStmt QueryValidDerivers;
     SQLiteStmt QueryDerivationOutputs;
     SQLiteStmt QueryRealisedOutput;
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
+    SQLiteStmt QueryRealisationReferences;
+    SQLiteStmt AddRealisationReference;
 };
 
 int getSchema(Path schemaPath)
@@ -76,7 +79,7 @@ int getSchema(Path schemaPath)
 
 void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 {
-    const int nixCASchemaVersion = 1;
+    const int nixCASchemaVersion = 2;
     int curCASchema = getSchema(schemaPath);
     if (curCASchema != nixCASchemaVersion) {
         if (curCASchema > nixCASchemaVersion) {
@@ -94,7 +97,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
               #include "ca-specific-schema.sql.gen.hh"
                 ;
             db.exec(schema);
+            curCASchema = nixCASchemaVersion;
         }
+
+        if (curCASchema < 2) {
+            SQLiteTxn txn(db);
+            // Ugly little sql dance to add a new `id` column and make it the primary key
+            db.exec(R"(
+                create table Realisations2 (
+                    id integer primary key autoincrement not null,
+                    drvPath text not null,
+                    outputName text not null, -- symbolic output id, usually "out"
+                    outputPath integer not null,
+                    signatures text, -- space-separated list
+                    foreign key (outputPath) references ValidPaths(id) on delete cascade
+                );
+                insert into Realisations2 (drvPath, outputName, outputPath, signatures)
+                    select drvPath, outputName, outputPath, signatures from Realisations;
+                drop table Realisations;
+                alter table Realisations2 rename to Realisations;
+            )");
+            db.exec(R"(
+                create index if not exists IndexRealisations on Realisations(drvPath, outputName);
+
+                create table if not exists RealisationsRefs (
+                    referrer integer not null,
+                    realisationReference integer,
+                    foreign key (referrer) references Realisations(id) on delete cascade,
+                    foreign key (realisationReference) references Realisations(id) on delete restrict
+                );
+            )");
+            txn.commit();
+        }
+
         writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
         lockFile(lockFd.get(), ltRead, true);
     }
@@ -106,9 +141,6 @@ LocalStore::LocalStore(const Params & params)
     , LocalStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
-    , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
-        "physical path to the Nix store"}
-    , realStoreDir(realStoreDir_)
     , dbDir(stateDir + "/db")
     , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
@@ -153,13 +185,13 @@ LocalStore::LocalStore(const Params & params)
             printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
         else {
             struct stat st;
-            if (stat(realStoreDir.c_str(), &st))
+            if (stat(realStoreDir.get().c_str(), &st))
                 throw SysError("getting attributes of path '%1%'", realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
-                if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
+                if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
                     throw SysError("changing ownership of path '%1%'", realStoreDir);
-                if (chmod(realStoreDir.c_str(), perm) == -1)
+                if (chmod(realStoreDir.get().c_str(), perm) == -1)
                     throw SysError("changing permissions on path '%1%'", realStoreDir);
             }
         }
@@ -314,9 +346,18 @@ LocalStore::LocalStore(const Params & params)
                 values (?, ?, (select id from ValidPaths where path = ?), ?)
                 ;
             )");
+        state->stmts->UpdateRealisedOutput.create(state->db,
+            R"(
+                update Realisations
+                    set signatures = ?
+                where
+                    drvPath = ? and
+                    outputName = ?
+                ;
+            )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
-                select Output.path, Realisations.signatures from Realisations
+                select Realisations.id, Output.path, Realisations.signatures from Realisations
                     inner join ValidPaths as Output on Output.id = Realisations.outputPath
                     where drvPath = ? and outputName = ?
                     ;
@@ -328,6 +369,19 @@ LocalStore::LocalStore(const Params & params)
                     where drvPath = ?
                     ;
             )");
+        state->stmts->QueryRealisationReferences.create(state->db,
+            R"(
+                select drvPath, outputName from Realisations
+                    join RealisationsRefs on realisationReference = Realisations.id
+                    where referrer = ?;
+            )");
+        state->stmts->AddRealisationReference.create(state->db,
+            R"(
+                insert or replace into RealisationsRefs (referrer, realisationReference)
+                values (
+                    ?,
+                    (select id from Realisations where drvPath = ? and outputName = ?));
+            )");
     }
 }
 
@@ -437,14 +491,14 @@ void LocalStore::makeStoreWritable()
     if (getuid() != 0) return;
     /* Check if /nix/store is on a read-only mount. */
     struct statvfs stat;
-    if (statvfs(realStoreDir.c_str(), &stat) != 0)
+    if (statvfs(realStoreDir.get().c_str(), &stat) != 0)
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
-        if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
+        if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
             throw SysError("remounting %1% writable", realStoreDir);
     }
 #endif
@@ -664,14 +718,54 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
     settings.requireExperimentalFeature("ca-derivations");
-    auto state(_state.lock());
     retrySQLite<void>([&]() {
-        state->stmts->RegisterRealisedOutput.use()
-            (info.id.strHash())
-            (info.id.outputName)
-            (printStorePath(info.outPath))
-            (concatStringsSep(" ", info.signatures))
-            .exec();
+        auto state(_state.lock());
+        if (auto oldR = queryRealisation_(*state, info.id)) {
+            if (info.isCompatibleWith(*oldR)) {
+                auto combinedSignatures = oldR->signatures;
+                combinedSignatures.insert(info.signatures.begin(),
+                    info.signatures.end());
+                state->stmts->UpdateRealisedOutput.use()
+                    (concatStringsSep(" ", combinedSignatures))
+                    (info.id.strHash())
+                    (info.id.outputName)
+                    .exec();
+            } else {
+                throw Error("Trying to register a realisation of '%s', but we already "
+                            "have another one locally.\n"
+                            "Local:  %s\n"
+                            "Remote: %s",
+                    info.id.to_string(),
+                    printStorePath(oldR->outPath),
+                    printStorePath(info.outPath)
+                );
+            }
+        } else {
+            state->stmts->RegisterRealisedOutput.use()
+                (info.id.strHash())
+                (info.id.outputName)
+                (printStorePath(info.outPath))
+                (concatStringsSep(" ", info.signatures))
+                .exec();
+        }
+        uint64_t myId = state->db.getLastInsertedRowId();
+        for (auto & [outputId, depPath] : info.dependentRealisations) {
+            auto localRealisation = queryRealisationCore_(*state, outputId);
+            if (!localRealisation)
+                throw Error("unable to register the derivation '%s' as it "
+                            "depends on the non existent '%s'",
+                    info.id.to_string(), outputId.to_string());
+            if (localRealisation->second.outPath != depPath)
+                throw Error("unable to register the derivation '%s' as it "
+                            "depends on a realisation of '%s' that doesn’t"
+                            "match what we have locally",
+                    info.id.to_string(), outputId.to_string());
+            state->stmts->AddRealisationReference.use()
+                (myId)
+                (outputId.strHash())
+                (outputId.outputName)
+                .exec();
+        }
     });
 }
 
@@ -1682,22 +1776,68 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
     }
 }
 
-std::optional<const Realisation> LocalStore::queryRealisation(
-    const DrvOutput& id) {
-    typedef std::optional<const Realisation> Ret;
-    return retrySQLite<Ret>([&]() -> Ret {
-        auto state(_state.lock());
-        auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
-            id.outputName));
-        if (!use.next())
-            return std::nullopt;
-        auto outputPath = parseStorePath(use.getStr(0));
-        auto signatures = tokenizeString<StringSet>(use.getStr(1));
-        return Ret{Realisation{
-            .id = id, .outPath = outputPath, .signatures = signatures}};
-    });
+std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_(
+        LocalStore::State & state,
+        const DrvOutput & id)
+{
+    auto useQueryRealisedOutput(
+            state.stmts->QueryRealisedOutput.use()
+                (id.strHash())
+                (id.outputName));
+    if (!useQueryRealisedOutput.next())
+        return std::nullopt;
+    auto realisationDbId = useQueryRealisedOutput.getInt(0);
+    auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
+    auto signatures =
+        tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
+
+    return {{
+        realisationDbId,
+        Realisation{
+            .id = id,
+            .outPath = outputPath,
+            .signatures = signatures,
+        }
+    }};
 }
 
+std::optional<const Realisation> LocalStore::queryRealisation_(
+            LocalStore::State & state,
+            const DrvOutput & id)
+{
+    auto maybeCore = queryRealisationCore_(state, id);
+    if (!maybeCore)
+        return std::nullopt;
+    auto [realisationDbId, res] = *maybeCore;
+
+    std::map<DrvOutput, StorePath> dependentRealisations;
+    auto useRealisationRefs(
+        state.stmts->QueryRealisationReferences.use()
+            (realisationDbId));
+    while (useRealisationRefs.next()) {
+        auto depId = DrvOutput {
+            Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
+            useRealisationRefs.getStr(1),
+        };
+        auto dependentRealisation = queryRealisationCore_(state, depId);
+        assert(dependentRealisation); // Enforced by the db schema
+        auto outputPath = dependentRealisation->second.outPath;
+        dependentRealisations.insert({depId, outputPath});
+    }
+
+    res.dependentRealisations = dependentRealisations;
+
+    return { res };
+}
+
+std::optional<const Realisation>
+LocalStore::queryRealisation(const DrvOutput & id)
+{
+    return retrySQLite<std::optional<const Realisation>>([&]() {
+        auto state(_state.lock());
+        return queryRealisation_(*state, id);
+    });
+}
 
 FixedOutputHash LocalStore::hashCAPath(
     const FileIngestionMethod & method, const HashType & hashType,

src/libstore/local-store.hh

@@ -83,9 +83,6 @@ private:
 
 public:
 
-    PathSetting realStoreDir_;
-
-    const Path realStoreDir;
     const Path dbDir;
     const Path linksDir;
     const Path reservedPath;
@@ -206,6 +203,8 @@ public:
     void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
     void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
 
+    std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
+    std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
 
 private:
@@ -279,8 +278,6 @@ private:
     void signPathInfo(ValidPathInfo & info);
     void signRealisation(Realisation &);
 
-    Path getRealStoreDir() override { return realStoreDir; }
-
     void createUser(const std::string & userName, uid_t userId) override;
 
     // XXX: Make a generic `Store` method

src/libstore/misc.cc

@@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(i);
 
             if (includeDerivers && path.isDerivation())
-                for (auto& i : queryDerivationOutputs(path))
-                    if (isValidPath(i) && queryPathInfo(i)->deriver == path)
-                        res.insert(i);
+                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
+                    if (maybeOutPath && isValidPath(*maybeOutPath))
+                        res.insert(*maybeOutPath);
             return res;
         };
     else
@@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(ref);
 
             if (includeOutputs && path.isDerivation())
-                for (auto& i : queryDerivationOutputs(path))
-                    if (isValidPath(i))
-                        res.insert(i);
+                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
+                    if (maybeOutPath && isValidPath(*maybeOutPath))
+                        res.insert(*maybeOutPath);
 
             if (includeDerivers && info->deriver && isValidPath(*info->deriver))
                 res.insert(*info->deriver);
@@ -254,5 +254,44 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         }});
 }
 
+std::map<DrvOutput, StorePath> drvOutputReferences(
+    const std::set<Realisation> & inputRealisations,
+    const StorePathSet & pathReferences)
+{
+    std::map<DrvOutput, StorePath> res;
 
+    for (const auto & input : inputRealisations) {
+        if (pathReferences.count(input.outPath)) {
+            res.insert({input.id, input.outPath});
+        }
+    }
+
+    return res;
+}
+
+std::map<DrvOutput, StorePath> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath)
+{
+    std::set<Realisation> inputRealisations;
+
+    for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
+        auto outputHashes =
+            staticOutputHashes(store, store.readDerivation(inputDrv));
+        for (const auto& outputName : outputNames) {
+            auto thisRealisation = store.queryRealisation(
+                DrvOutput{outputHashes.at(outputName), outputName});
+            if (!thisRealisation)
+                throw Error(
+                    "output '%s' of derivation '%s' isn’t built", outputName,
+                    store.printStorePath(inputDrv));
+            inputRealisations.insert(*thisRealisation);
+        }
+    }
+
+    auto info = store.queryPathInfo(outputPath);
+
+    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
+}
 }

src/libstore/optimise-store.cc

@@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
        permissions). */
-    bool mustToggle = dirOf(path) != realStoreDir;
+    bool mustToggle = dirOf(path) != realStoreDir.get();
     if (mustToggle) makeWritable(dirOf(path));
 
     /* When we're done, make the directory read-only again and reset

src/libstore/parsed-derivations.cc

@@ -91,6 +91,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
     StringSet res;
     for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
         res.insert(i);
+    if (!derivationHasKnownOutputPaths(drv.type()))
+        res.insert("ca-derivations");
     return res;
 }
 

src/libstore/realisation.cc

@@ -1,5 +1,6 @@
 #include "realisation.hh"
 #include "store-api.hh"
+#include "closure.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -21,11 +22,52 @@ std::string DrvOutput::to_string() const {
     return strHash() + "!" + outputName;
 }
 
+std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
+{
+    std::set<Realisation> res;
+    Realisation::closure(store, startOutputs, res);
+    return res;
+}
+
+void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
+{
+    auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
+        std::set<Realisation> res;
+        for (auto& [currentDep, _] : current.dependentRealisations) {
+            if (auto currentRealisation = store.queryRealisation(currentDep))
+                res.insert(*currentRealisation);
+            else
+                throw Error(
+                    "Unrealised derivation '%s'", currentDep.to_string());
+        }
+        return res;
+    };
+
+    computeClosure<Realisation>(
+        startOutputs, res,
+        [&](const Realisation& current,
+            std::function<void(std::promise<std::set<Realisation>>&)>
+                processEdges) {
+            std::promise<std::set<Realisation>> promise;
+            try {
+                auto res = getDeps(current);
+                promise.set_value(res);
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+            return processEdges(promise);
+        });
+}
+
 nlohmann::json Realisation::toJSON() const {
+    auto jsonDependentRealisations = nlohmann::json::object();
+    for (auto & [depId, depOutPath] : dependentRealisations)
+        jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
     return nlohmann::json{
         {"id", id.to_string()},
         {"outPath", outPath.to_string()},
         {"signatures", signatures},
+        {"dependentRealisations", jsonDependentRealisations},
     };
 }
 
@@ -51,10 +93,16 @@ Realisation Realisation::fromJSON(
     if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
         signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
 
+    std::map <DrvOutput, StorePath> dependentRealisations;
+    if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end())
+        for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>())
+            dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
+
     return Realisation{
         .id = DrvOutput::parse(getField("id")),
         .outPath = StorePath(getField("outPath")),
         .signatures = signatures,
+        .dependentRealisations = dependentRealisations,
     };
 }
 
@@ -92,6 +140,16 @@ StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }
 
+bool Realisation::isCompatibleWith(const Realisation & other) const
+{
+    assert (id == other.id);
+    if (outPath == other.outPath) {
+        assert(dependentRealisations == other.dependentRealisations);
+        return true;
+    }
+    return false;
+}
+
 void RealisedPath::closure(
     Store& store,
     const RealisedPath::Set& startPaths,

src/libstore/realisation.hh

@@ -28,6 +28,14 @@ struct Realisation {
 
     StringSet signatures;
 
+    /**
+     * The realisations that are required for the current one to be valid.
+     *
+     * When importing this realisation, the store will first check that all its
+     * dependencies exist, and map to the correct output path
+     */
+    std::map<DrvOutput, StorePath> dependentRealisations;
+
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
@@ -36,6 +44,11 @@ struct Realisation {
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 
+    static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
+    static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
+
+    bool isCompatibleWith(const Realisation & other) const;
+
     StorePath getPath() const { return outPath; }
 
     GENERATE_CMP(Realisation, me->id, me->outPath);

src/libstore/remote-store.cc

@@ -653,8 +653,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
     conn->to << wopRegisterDrvOutput;
-    conn->to << info.id.to_string();
-    conn->to << std::string(info.outPath.to_string());
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        conn->to << info.id.to_string();
+        conn->to << std::string(info.outPath.to_string());
+    } else {
+        worker_proto::write(*this, conn->to, info);
+    }
     conn.processStderr();
 }
 
@@ -664,10 +668,17 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();
-    auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
-    if (outPaths.empty())
-        return std::nullopt;
-    return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+        auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
+        if (outPaths.empty())
+            return std::nullopt;
+        return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+    } else {
+        auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
+        if (realisations.empty())
+            return std::nullopt;
+        return *realisations.begin();
+    }
 }
 
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)

src/libstore/store-api.cc

@@ -337,6 +337,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     return info;
 }
 
+StringSet StoreConfig::getDefaultSystemFeatures()
+{
+    auto res = settings.systemFeatures.get();
+    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+        res.insert("ca-derivations");
+    return res;
+}
 
 Store::Store(const Params & params)
     : StoreConfig(params)
@@ -780,20 +787,39 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
     StorePathSet storePaths;
-    std::set<Realisation> realisations;
+    std::set<Realisation> toplevelRealisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto realisation = std::get_if<Realisation>(&path.raw)) {
             settings.requireExperimentalFeature("ca-derivations");
-            realisations.insert(*realisation);
+            toplevelRealisations.insert(*realisation);
         }
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+
+    ThreadPool pool;
+
     try {
-        for (auto & realisation : realisations) {
-            dstStore->registerDrvOutput(realisation, checkSigs);
-        }
-    } catch (MissingExperimentalFeature & e) {
+        // Copy the realisation closure
+        processGraph<Realisation>(
+            pool, Realisation::closure(*srcStore, toplevelRealisations),
+            [&](const Realisation& current) -> std::set<Realisation> {
+                std::set<Realisation> children;
+                for (const auto& [drvOutput, _] : current.dependentRealisations) {
+                    auto currentChild = srcStore->queryRealisation(drvOutput);
+                    if (!currentChild)
+                        throw Error(
+                            "Incomplete realisation closure: '%s' is a "
+                            "dependency of '%s' but isn’t registered",
+                            drvOutput.to_string(), current.id.to_string());
+                    children.insert(*currentChild);
+                }
+                return children;
+            },
+            [&](const Realisation& current) -> void {
+                dstStore->registerDrvOutput(current, checkSigs);
+            });
+    } catch (MissingExperimentalFeature& e) {
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
         // to at least copy the output paths.

src/libstore/store-api.hh

@@ -180,6 +180,8 @@ struct StoreConfig : public Config
 
     StoreConfig() = delete;
 
+    StringSet getDefaultSystemFeatures();
+
     virtual ~StoreConfig() { }
 
     virtual const std::string name() = 0;
@@ -196,7 +198,7 @@ struct StoreConfig : public Config
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
 
-    Setting<StringSet> systemFeatures{this, settings.systemFeatures,
+    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
         "system-features",
         "Optional features that the system this store builds on implements (like \"kvm\")."};
 
@@ -864,4 +866,9 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
 
 std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
 
+std::map<DrvOutput, StorePath> drvOutputReferences(
+    Store & store,
+    const Derivation & drv,
+    const StorePath & outputPath);
+
 }

src/libstore/worker-protocol.hh

@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 30)
+#define PROTOCOL_VERSION (1 << 8 | 31)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

src/libutil/comparator.hh

@@ -25,6 +25,8 @@
     }
 #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
 #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
     GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args)
+    GENERATE_LEQ(args) \
+    GENERATE_NEQ(args)

src/nix-build/nix-build.cc

@@ -387,6 +387,12 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
+        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+            auto resolvedDrv = drv.tryResolve(*store);
+            assert(resolvedDrv && "Successfully resolved the derivation");
+            drv = *resolvedDrv;
+        }
+
         // Set the environment.
         auto env = getEnv();
 

src/nix/build.cc

@@ -69,8 +69,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                             store2->addPermRoot(bo.path, absPath(symlink));
                         },
                         [&](BuiltPath::Built bfd) {
-                            auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
-                            for (auto & output : builtOutputs) {
+                            for (auto & output : bfd.outputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);

src/nix/develop.cc

@@ -144,17 +144,26 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs) {
-        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
-        drv.env[output.first] = "";
-    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        for (auto & output : drv.outputs) {
+            output.second = {
+                .output = DerivationOutputDeferred{},
+            };
+            drv.env[output.first] = hashPlaceholder(output.first);
+        }
+    } else {
+        for (auto & output : drv.outputs) {
+            output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+            drv.env[output.first] = "";
+        }
+        Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
 
-    for (auto & output : drv.outputs) {
-        auto outPath = store->makeOutputPath(output.first, h, drv.name);
-        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
-        drv.env[output.first] = store->printStorePath(outPath);
+        for (auto & output : drv.outputs) {
+            auto outPath = store->makeOutputPath(output.first, h, drv.name);
+            output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+            drv.env[output.first] = store->printStorePath(outPath);
+        }
     }
 
     auto shellDrvPath = writeDerivation(*store, drv);
@@ -162,8 +171,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Build the derivation. */
     store->buildPaths({DerivedPath::Built{shellDrvPath}});
 
-    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
-        auto & [_1, optPath] = outputAndOptPath;
+    for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) {
         assert(optPath);
         auto & outPath = *optPath;
         assert(store->isValidPath(outPath));
@@ -185,6 +193,7 @@ struct Common : InstallableCommand, MixProfile
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",
+        "NIX_REMOTE",
         "PPID",
         "PWD",
         "SHELLOPTS",

src/nix/profile-remove.md

@@ -15,6 +15,7 @@ R""(
   ```
 
 * Remove all packages:
+
   ```console
   # nix profile remove '.*'
   ```

src/nix/run.cc

@@ -43,8 +43,8 @@ struct RunCommon : virtual Command
            helper program (chrootHelper() below) to do the work. */
         auto store2 = store.dynamic_pointer_cast<LocalStore>();
 
-        if (store2 && store->storeDir != store2->realStoreDir) {
-            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
+        if (store2 && store->storeDir != store2->getRealStoreDir()) {
+            Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
             for (auto & arg : args) helperArgs.push_back(arg);
 
             execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());

src/nix/search.cc

@@ -9,7 +9,6 @@
 #include "shared.hh"
 #include "eval-cache.hh"
 #include "attr-path.hh"
-#include "value.hh"
 
 #include <regex>
 #include <fstream>
@@ -82,46 +81,31 @@ struct CmdSearch : InstallableCommand, MixJSON
 
         uint64_t results = 0;
 
-        std::function<void(Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
+        std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
 
-        Value * vTmp = state->allocValue();
-
-        visit = [&](Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)
+        visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
-            auto recurse = [&]()
-            {
-                for (auto & attrName : state->getFields(current, noPos)) {
-                  try {
-                    auto attrPath2(attrPath);
-                    attrPath2.push_back(attrName);
-                    auto attrValue = state->allocValue();
-                    auto value_ = allocRootValue(attrValue);
-                    auto childField = state->lazyGetAttrField(current, {attrName}, noPos,
-                                            *attrValue);
-                    if (childField != EvalState::LazyValueType::DelayedUncacheable)
-                        visit(*attrValue, attrPath2, false);
-                  } catch (EvalError &e) {
-                    if (!(attrPath.size() > 0 &&
-                          attrPath[0] == "legacyPackages"))
-                      throw;
-                  }
-                }
-            };
-
             try {
-                auto maybeTypeField = state->lazyGetAttrField(current, {state->sType}, noPos, *vTmp);
-                if (maybeTypeField == EvalState::LazyValueType::PlainValue
-                    && vTmp->type() == nix::nString
-                    && strcmp(vTmp->string.s, "derivation") == 0) {
+                auto recurse = [&]()
+                {
+                    for (const auto & attr : cursor.getAttrs()) {
+                        auto cursor2 = cursor.getAttr(attr);
+                        auto attrPath2(attrPath);
+                        attrPath2.push_back(attr);
+                        visit(*cursor2, attrPath2, false);
+                    }
+                };
+
+                if (cursor.isDerivation()) {
                     size_t found = 0;
 
-                    state->getAttrFieldThrow(current, {state->sName}, noPos, *vTmp);
-                    DrvName name(state->forceString(*vTmp));
+                    DrvName name(cursor.getAttr("name")->getString());
 
-                    auto hasDescription = state->getAttrField(current, {state->sMeta, state->sDescription}, noPos, *vTmp);
-                    auto description = hasDescription ? state->forceString(*vTmp) : "";
+                    auto aMeta = cursor.maybeGetAttr("meta");
+                    auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
+                    auto description = aDescription ? aDescription->getString() : "";
                     std::replace(description.begin(), description.end(), '\n', ' ');
                     auto attrPath2 = concatStringsSep(".", attrPath);
 
@@ -134,8 +118,8 @@ struct CmdSearch : InstallableCommand, MixJSON
                         std::regex_search(name.name, nameMatch, regex);
                         std::regex_search(description, descriptionMatch, regex);
                         if (!attrPathMatch.empty()
-                                || !nameMatch.empty()
-                                || !descriptionMatch.empty())
+                            || !nameMatch.empty()
+                            || !descriptionMatch.empty())
                             found++;
                     }
 
@@ -150,37 +134,39 @@ struct CmdSearch : InstallableCommand, MixJSON
                             auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
                             if (results > 1) logger->cout("");
                             logger->cout(
-                                    "* %s%s",
-                                    wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
-                                    name.version != "" ? " (" + name.version + ")" : "");
+                                "* %s%s",
+                                wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
+                                name.version != "" ? " (" + name.version + ")" : "");
                             if (description != "")
                                 logger->cout(
-                                        "  %s", hilite(description, descriptionMatch, ANSI_NORMAL));
+                                    "  %s", hilite(description, descriptionMatch, ANSI_NORMAL));
                         }
                     }
-                } else if (
-                        attrPath.size() == 0
-                        || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
-                        || (attrPath[0] == "packages" && attrPath.size() <= 2))
+                }
+
+                else if (
+                    attrPath.size() == 0
+                    || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
+                    || (attrPath[0] == "packages" && attrPath.size() <= 2))
                     recurse();
 
                 else if (initialRecurse)
                     recurse();
 
                 else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
-                    auto recurseFieldInfo = state->lazyGetAttrField(current, {state->sRecurseForDerivations}, noPos, *vTmp);
-                    auto hasRecurse = recurseFieldInfo == EvalState::LazyValueType::PlainValue;
-                    if (hasRecurse && state->forceBool(*vTmp, noPos))
+                    auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
+                    if (attr && attr->getBool())
                         recurse();
                 }
+
             } catch (EvalError & e) {
                 if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
                     throw;
             }
         };
 
-        for (auto & [value, pos, prefix] : installable->toValues(*state))
-            visit(*value, parseAttrPath(*state, prefix), true);
+        for (auto & [cursor, prefix] : installable->getCursors(*state))
+            visit(*cursor, parseAttrPath(*state, prefix), true);
 
         if (!json && !results)
             throw Error("no results for the given search term(s)!");

tests/build-remote-content-addressed-floating.sh

@@ -4,4 +4,6 @@ file=build-hook-ca-floating.nix
 
 sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
 
+CONTENT_ADDRESSED=true
+
 source build-remote.sh

tests/build-remote.sh

@@ -6,12 +6,17 @@ unset NIX_STATE_DIR
 
 function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
 
+EXTRA_SYSTEM_FEATURES=()
+if [[ -n "$CONTENT_ADDRESSED" ]]; then
+    EXTRA_SYSTEM_FEATURES=("ca-derivations")
+fi
+
 builders=(
   # system-features will automatically be added to the outer URL, but not inner
   # remote-store URL.
-  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
-  "$TEST_ROOT/machine2 - - 1 1 bar"
-  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
+  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})"
+  "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})"
+  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})"
 )
 
 chmod -R +w $TEST_ROOT/machine* || true

tests/build.sh

@@ -1,7 +1,7 @@
 source common.sh
 
 expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
-nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
+nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
     (.outputs.first | match(".*multiple-outputs-a-first")) and
@@ -10,10 +10,10 @@ nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
     (.drvPath | match(".*multiple-outputs-b.drv")) and
     (.outputs.out | match(".*multiple-outputs-b")))
 '
-
 testNormalization () {
     clearStore
-    outPath=$(nix-build ./simple.nix)
+    outPath=$(nix-build ./simple.nix --no-out-link)
     test "$(stat -c %Y $outPath)" -eq 1
 }
+
 testNormalization

tests/ca/build-with-garbage-path.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Regression test for https://github.com/NixOS/nix/issues/4858
+
+source common.sh
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+# Get the output path of `rootCA`, and put some garbage instead
+outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"
+nix-store --delete "$outPath"
+touch "$outPath"
+
+# The build should correctly remove the garbage and put the expected path instead
+nix-build ./content-addressed.nix -A rootCA --no-out-link
+
+# Rebuild it. This shouldn’t overwrite the existing path
+oldInode=$(stat -c '%i' "$outPath")
+nix-build ./content-addressed.nix -A rootCA --no-out-link --arg seed 2
+newInode=$(stat -c '%i' "$outPath")
+[[ "$oldInode" == "$newInode" ]]

tests/ca/duplicate-realisation-in-closure.sh

@@ -0,0 +1,26 @@
+source ./common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+rm -rf $REMOTE_STORE_DIR
+clearStore
+
+# Build dep1 and push that to the binary cache.
+# This entails building (and pushing) current-time.
+nix copy --to "$REMOTE_STORE" -f nondeterministic.nix dep1
+clearStore
+sleep 2 # To make sure that `$(date)` will be different
+# Build dep2.
+# As we’ve cleared the cache, we’ll have to rebuild current-time. And because
+# the current time isn’t the same as before, this will yield a new (different)
+# realisation
+nix build -f nondeterministic.nix dep2 --no-link
+
+# Build something that depends both on dep1 and dep2.
+# If everything goes right, we should rebuild dep2 rather than fetch it from
+# the cache (because that would mean duplicating `current-time` in the closure),
+# and have `dep1 == dep2`.
+nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link

tests/ca/gc.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Ensure that garbage collection works properly with ca derivations
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+
+cd ..
+source gc.sh

tests/ca/nix-shell.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+CONTENT_ADDRESSED=true
+cd ..
+source ./nix-shell.sh
+

tests/ca/nondeterministic.nix

@@ -0,0 +1,35 @@
+with import ./config.nix;
+
+let mkCADerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+} // args);
+in
+
+rec {
+  currentTime = mkCADerivation {
+    name = "current-time";
+    buildCommand = ''
+      mkdir $out
+      echo $(date) > $out/current-time
+    '';
+  };
+  dep = seed: mkCADerivation {
+    name = "dep";
+    inherit seed;
+    buildCommand = ''
+      echo ${currentTime} > $out
+    '';
+  };
+  dep1 = dep 1;
+  dep2 = dep 2;
+  toplevel = mkCADerivation {
+    name = "toplevel";
+    buildCommand = ''
+      test ${dep1} == ${dep2}
+      touch $out
+    '';
+  };
+}
+

tests/ca/post-hook.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+cd ..
+source ./post-hook.sh
+
+

tests/ca/recursive.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+cd ..
+source ./recursive.sh
+
+

tests/ca/substitute.sh

@@ -17,11 +17,15 @@ buildDrvs () {
 
 # Populate the remote cache
 clearStore
-buildDrvs --post-build-hook ../push-to-store.sh
+nix copy --to $REMOTE_STORE --file ./content-addressed.nix
 
 # Restart the build on an empty store, ensuring that we don't build
 clearStore
-buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA
+# Check that the thing we’ve just substituted has its realisation stored
+nix realisation info --file ./content-addressed.nix transitivelyDependentCA
+# Check that its dependencies have it too
+nix realisation info --file ./content-addressed.nix dependentCA rootCA
 
 # Same thing, but
 # 1. With non-ca derivations

tests/config.nix.in

@@ -1,3 +1,12 @@
+let
+  contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1";
+  caArgs = if contentAddressedByDefault then {
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } else {};
+in
+
 rec {
   shell = "@bash@";
 
@@ -13,6 +22,6 @@ rec {
       builder = shell;
       args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
-    } // removeAttrs args ["builder" "meta"])
+    } // caArgs // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 }

tests/gc.sh

@@ -12,7 +12,7 @@ ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo
 nix-store --gc --print-roots | grep $outPath
 nix-store --gc --print-live | grep $outPath
 nix-store --gc --print-dead | grep $drvPath
-if nix-store --gc --print-dead | grep $outPath; then false; fi
+if nix-store --gc --print-dead | grep -E $outPath$; then false; fi
 
 nix-store --gc --print-dead
 

tests/local.mk

@@ -2,6 +2,7 @@ nix_tests = \
   hash.sh lang.sh add.sh simple.sh dependencies.sh \
   config.sh \
   gc.sh \
+  ca/gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
@@ -38,6 +39,7 @@ nix_tests = \
   search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
+  ca/post-hook.sh \
   function-trace.sh \
   recursive.sh \
   describe-stores.sh \
@@ -45,9 +47,13 @@ nix_tests = \
   build.sh \
   compute-levels.sh \
   ca/build.sh \
+  ca/build-with-garbage-path.sh \
+  ca/duplicate-realisation-in-closure.sh \
   ca/substitute.sh \
   ca/signatures.sh \
+  ca/nix-shell.sh \
   ca/nix-run.sh \
+  ca/recursive.sh \
   ca/nix-copy.sh
   # parallel.sh
 

tests/nix-shell.sh

@@ -2,6 +2,20 @@ source common.sh
 
 clearStore
 
+if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
+    nix-shell () {
+        command nix-shell --arg contentAddressed true "$@"
+    }
+
+    nix_develop() {
+        nix develop --arg contentAddressed true "$@"
+    }
+else
+    nix_develop() {
+        nix develop "$@"
+    }
+fi
+
 # Test nix-shell -A
 export IMPURE_VAR=foo
 export SELECTED_IMPURE_VAR=baz
@@ -41,7 +55,7 @@ output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(f
 [ "$output" = "foo bar" ]
 
 # Test nix-shell shebang mode
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
 chmod a+rx $TEST_ROOT/shell.shebang.sh
 
 output=$($TEST_ROOT/shell.shebang.sh abc def)
@@ -49,7 +63,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def)
 
 # Test nix-shell shebang mode again with metacharacters in the filename.
 # First word of filename is chosen to not match any file in the test root.
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
@@ -58,7 +72,7 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
 # Test nix-shell shebang mode for ruby
 # This uses a fake interpreter that returns the arguments passed
 # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
-sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
 chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
@@ -66,20 +80,20 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test nix-shell shebang mode for ruby again with metacharacters in the filename.
 # Note: fake interpreter only space-separates args without adding escapes to its output.
-sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
 [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
 
 # Test 'nix develop'.
-nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
 
 # Ensure `nix develop -c` preserves stdin
 echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 
 # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
-nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo
+nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)

tests/post-hook.sh

@@ -4,7 +4,7 @@ clearStore
 
 rm -f $TEST_ROOT/result
 
-export REMOTE_STORE=$TEST_ROOT/remote_store
+export REMOTE_STORE=file:$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store
 nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh

tests/recursive.sh

@@ -9,9 +9,9 @@ rm -f $TEST_ROOT/result
 
 export unreachable=$(nix store add-path ./recursive.sh)
 
-NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
+NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
   with import ./config.nix;
-  mkDerivation {
+  mkDerivation rec {
     name = "recursive";
     dummy = builtins.toFile "dummy" "bla bla";
     SHELL = shell;
@@ -19,11 +19,13 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
     # Note: this is a string without context.
     unreachable = builtins.getEnv "unreachable";
 
+    NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT";
+
     requiredSystemFeatures = [ "recursive-nix" ];
 
     buildCommand = '\'\''
       mkdir $out
-      opts="--experimental-features nix-command"
+      opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}"
 
       PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
 
@@ -46,16 +48,15 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
       # Add it to our closure.
       ln -s $foobar $out/foobar
 
-      [[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
+      [[ $(nix $opts path-info --all | wc -l) -eq 4 ]]
 
       # Build a derivation.
       nix $opts build -L --impure --expr '\''
-        derivation {
+        with import ${./config.nix};
+        mkDerivation {
           name = "inner1";
-          builder = builtins.getEnv "SHELL";
-          system = builtins.getEnv "system";
+          buildCommand = "echo $fnord blaat > $out";
           fnord = builtins.toFile "fnord" "fnord";
-          args = [ "-c" "echo $fnord blaat > $out" ];
         }
       '\''
 

tests/shell.nix

@@ -1,6 +1,18 @@
-{ inNixShell ? false }:
+{ inNixShell ? false, contentAddressed ? false }:
 
-with import ./config.nix;
+let cfg = import ./config.nix; in
+with cfg;
+
+let
+  mkDerivation =
+    if contentAddressed then
+      args: cfg.mkDerivation ({
+        __contentAddressed = true;
+        outputHashMode = "recursive";
+        outputHashAlgo = "sha256";
+      } // args)
+    else cfg.mkDerivation;
+in
 
 let pkgs = rec {
   setupSh = builtins.toFile "setup" ''

thingToBench.nix

@@ -1,12 +0,0 @@
-let
-  # pkgs = (builtins.getFlake "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31").legacyPackages.x86_64-linux;
-  pkgs = import <nixpkgs> {};
-in
-  pkgs.runCommandNoCC "foo" {
-    buildInputs = [
-      pkgs.firefox
-      pkgs.pandoc
-      # pkgs.nixosTests.xfce
-    ];
-  }
-  "echo bar > $out"