Make nix search lazier

Only 4 evals left \o/

.github/workflows/test.yml

@@ -20,7 +20,8 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
+    #- run: nix flake check
+    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest

benchmark.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+set -euo pipefail
+set -x
+
+callNix () {
+    nix \
+        --experimental-features "nix-command flakes" \
+        --store /tmp/nix \
+        "$@"
+}
+
+callBuild () {
+    callNix \
+        eval --impure --file "$THINGTOBENCH" drvPath \
+        "$@"
+}
+getCompletions () {
+    NIX_GET_COMPLETIONS=6 callNix build "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31#firef" "$@"
+}
+runSearch () {
+    callNix search "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31" firefox "$@"
+}
+
+noCache () {
+    "$@" --option eval-cache false
+}
+coldCache () {
+    if [[ -e ~/.cache/nix/eval-cache-v2 || -e ~/.cache/nix/eval-cache-v3 ]]; then
+        echo "Error: The cache should be clean"
+        exit 1
+    fi
+    "$@"
+}
+
+run_all () {
+
+    mkdir -p "$out"
+
+    NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$out/eval-stats.json bash $0 noCache callBuild
+
+    hyperfine \
+        --warmup 2 \
+        --export-csv "$out/result.csv" \
+        --export-json "$out/result.json" \
+        --export-markdown "$out/result.md" \
+        --style basic \
+        --prepare '' "bash $0 noCache callBuild" \
+        --prepare 'rm -rf ~/.cache/nix/' "bash $0 coldCache callBuild" \
+        --prepare '' "bash $0 callBuild"
+}
+
+"$@"

configure.ac

@@ -65,7 +65,7 @@ AC_SYS_LARGEFILE
 AC_STRUCT_DIRENT_D_TYPE
 if test "$sys_name" = sunos; then
     # Solaris requires -lsocket -lnsl for network functions
-    LDFLAGS="-lsocket -lnsl $LDFLAGS"
+    LIBS="-lsocket -lnsl $LIBS"
 fi
 
 
@@ -150,7 +150,7 @@ int main() {
 }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
 AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
 if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
-    LDFLAGS="-latomic $LDFLAGS"
+    LIBS="-latomic $LIBS"
 fi
 
 PKG_PROG_PKG_CONFIG

doc/manual/src/command-ref/env-common.md

@@ -10,39 +10,35 @@ Most Nix commands interpret the following environment variables:
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
-
+    
         /home/eelco/Dev:/etc/nixos
-
+    
     will cause Nix to look for paths relative to `/home/eelco/Dev` and
     `/etc/nixos`, in this order. It is also possible to match paths
     against a prefix. For example, the value
-
+    
         nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
-
+    
     will cause Nix to search for `<nixpkgs/path>` in
     `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-
+    
     If a path in the Nix search path starts with `http://` or
     `https://`, it is interpreted as the URL of a tarball that will be
     downloaded and unpacked to a temporary location. The tarball must
     consist of a single top-level directory. For example, setting
     `NIX_PATH` to
-
-        nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
-
-    tells Nix to download and use the current contents of the
-    `master` branch in the `nixpkgs` repository.
-
-    The URLs of the tarballs from the official nixos.org channels (see
-    [the manual for `nix-channel`](nix-channel.md)) can be abbreviated
-    as `channel:<channel-name>`.  For instance, the following two
-    values of `NIX_PATH` are equivalent:
-
-        nixpkgs=channel:nixos-21.05
-        nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
-
-    The Nix search path can also be extended using the `-I` option to
-    many Nix commands, which takes precedence over `NIX_PATH`.
+    
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
+    
+    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
+    channel.
+    
+    A following shorthand can be used to refer to the official channels:
+    
+        nixpkgs=channel:nixos-15.09
+    
+    The search path can be extended using the `-I` option, which takes
+    precedence over `NIX_PATH`.
 
   - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
@@ -54,7 +50,7 @@ Most Nix commands interpret the following environment variables:
     builds are deployed to machines where `/nix/store` resolves
     differently. If you are sure that you’re not going to do that, you
     can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
-
+    
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
@@ -63,7 +59,7 @@ Most Nix commands interpret the following environment variables:
     $ mkdir /nix
     $ mount -o bind /mnt/otherdisk/nix /nix
     ```
-
+    
     Consult the mount 8 manual page for details.
 
   - `NIX_STORE_DIR`\

flake.lock

@@ -19,11 +19,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1624862269,
-        "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
+        "lastModified": 1622593737,
+        "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
+        "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e",
         "type": "github"
       },
       "original": {

flake.nix

@@ -229,7 +229,7 @@
             echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
           '';
 
-          doInstallCheck = true;
+          doInstallCheck = false;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           separateDebugInfo = true;
@@ -256,8 +256,7 @@
                 boost
                 nlohmann_json
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
+              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -295,6 +294,13 @@
             '';
         };
 
+        nix-benchmarks = prev.runCommandNoCC "nix-benchmarks" {
+          buildInputs = [ final.nix prev.hyperfine ];
+          THINGTOBENCH = ./thingToBench.nix;
+          NIX_PATH="nixpkgs=${prev.path}";
+        } ''
+          bash ${./benchmark.sh} run_all
+        '';
       };
 
       hydraJobs = {
@@ -486,7 +492,7 @@
       });
 
       packages = forAllSystems (system: {
-        inherit (nixpkgsFor.${system}) nix;
+        inherit (nixpkgsFor.${system}) nix nix-benchmarks;
       } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;

misc/zsh/completion.zsh

@@ -1,5 +1,3 @@
-#compdef nix
-
 function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
@@ -20,4 +18,4 @@ function _nix() {
   _describe 'nix' suggestions
 }
 
-_nix "$@"
+compdef _nix nix

src/build-remote/build-remote.cc

@@ -277,16 +277,7 @@ connected:
 
         auto drv = store->readDerivation(*drvPath);
         auto outputHashes = staticOutputHashes(*store, drv);
-
-        // Hijack the inputs paths of the derivation to include all the paths
-        // that come from the `inputDrvs` set.
-        // We don’t do that for the derivations whose `inputDrvs` is empty
-        // because
-        // 1. It’s not needed
-        // 2. Changing the `inputSrcs` set changes the associated output ids,
-        //  which break CA derivations
-        if (!drv.inputDrvs.empty())
-            drv.inputSrcs = store->parseStorePathSet(inputs);
+        drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
 

src/libcmd/installables.cc

@@ -302,6 +302,14 @@ Installable::getCursors(EvalState & state)
     return {{evalCache->getRoot(), ""}};
 }
 
+std::pair<Value*, Pos> Installable::toValue(EvalState & state)
+{
+    auto values = toValues(state);
+    if (values.empty())
+        throw Error("cannot find flake attribute '%s'", what());
+    return {std::get<0>(values[0]), std::get<1>(values[0])};
+}
+
 std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
 Installable::getCursor(EvalState & state)
 {
@@ -378,11 +386,11 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    std::pair<Value *, Pos> toValue(EvalState & state) override
+    std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override
     {
         auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
         state.forceValue(*vRes);
-        return {vRes, pos};
+        return {{vRes, pos, attrPath}};
     }
 
     virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -428,12 +436,10 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
     callFlake(state, lockedFlake, *vFlake);
 
-    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
-    assert(aOutputs);
-
-    state.forceValue(*aOutputs->value);
-
-    return aOutputs->value;
+    auto vRes = state.allocValue();
+    auto gotField = state.lazyGetAttrField(*vFlake, {state.symbols.create("outputs")}, noPos, *vRes);
+    assert(gotField != EvalState::LazyValueType::Missing);
+    return vRes;
 }
 
 ref<eval_cache::EvalCache> openEvalCache(
@@ -500,25 +506,34 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
-        auto attr = root->findAlongAttrPath(
-            parseAttrPath(*state, attrPath),
-            true
-        );
+        auto emptyArgs = state->allocBindings(0);
+        try {
+            auto [drvValue, pos] = findAlongAttrPath(
+                *state,
+                attrPath,
+                *emptyArgs,
+                *getFlakeOutputs(*state, *lockedFlake)
+            );
+            Value * v = state->allocValue();
+            if (!state->getAttrField(*drvValue, {state->sDrvPath}, pos, *v))
+                break;
+            auto drvPath = state->forceString(*v);
+            if (!state->getAttrField(*drvValue, {state->sOutPath}, pos, *v))
+                break;
+            auto outPath = state->forceString(*v);
+            if (!state->getAttrField(*drvValue, {state->sOutputName}, pos, *v))
+                break;
+            auto outputName = state->forceString(*v);
 
-        if (!attr) continue;
+            auto drvInfo = DerivationInfo{
+                state->store->parseStorePath(drvPath),
+                state->store->maybeParseStorePath(outPath),
+                outputName
+            };
 
-        if (!attr->isDerivation())
-            throw Error("flake output attribute '%s' is not a derivation", attrPath);
-
-        auto drvPath = attr->forceDerivation();
-
-        auto drvInfo = DerivationInfo{
-            std::move(drvPath),
-            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
-            attr->getAttr(state->sOutputName)->getString()
-        };
-
-        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
+            return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
+        } catch (AttrPathNotFound & e) {
+        }
     }
 
     throw Error("flake '%s' does not provide attribute %s",
@@ -532,25 +547,22 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
     return res;
 }
 
-std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
+std::vector<std::tuple<Value *, Pos, std::string>>
+InstallableFlake::toValues(EvalState & state)
 {
-    auto lockedFlake = getLockedFlake();
-
-    auto vOutputs = getFlakeOutputs(state, *lockedFlake);
-
+    auto vOutputs = getFlakeOutputs(state, *getLockedFlake());
     auto emptyArgs = state.allocBindings(0);
 
+    std::vector<std::tuple<Value *, Pos, std::string>> res;
+
     for (auto & attrPath : getActualAttrPaths()) {
         try {
             auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
-            state.forceValue(*v);
-            return {v, pos};
-        } catch (AttrPathNotFound & e) {
-        }
+            res.push_back({v, pos, attrPath});
+        } catch (Error &) {}
     }
 
-    throw Error("flake '%s' does not provide attribute %s",
-        flakeRef, showAttrPaths(getActualAttrPaths()));
+    return res;
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>

src/libcmd/installables.hh

@@ -41,10 +41,11 @@ struct Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual std::pair<Value *, Pos> toValue(EvalState & state)
+    virtual std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }
+    std::pair<Value *, Pos> toValue(EvalState & state);
 
     /* Return a value only if this installable is a store path or a
        symlink to it. */
@@ -109,7 +110,7 @@ struct InstallableFlake : InstallableValue
 
     std::vector<DerivationInfo> toDerivations() override;
 
-    std::pair<Value *, Pos> toValue(EvalState & state) override;
+    std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override;
 
     std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
     getCursors(EvalState & state) override;

src/libexpr/attr-path.cc

@@ -58,26 +58,23 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         Value * vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
-        state.forceValue(*v);
 
         /* It should evaluate to either a set or an expression,
            according to what is specified in the attrPath. */
 
         if (!attrIndex) {
 
-            if (v->type() != nAttrs)
-                throw TypeError(
-                    "the expression selected by the selection path '%1%' should be a set but is %2%",
-                    attrPath,
-                    showType(*v));
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v->attrs->end())
+            auto v2 = state.allocValue();
+            auto gotField = state.lazyGetAttrField(*v, {state.symbols.create(attr)}, pos, *v2) != EvalState::LazyValueType::Missing;
+            if (!gotField)
                 throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
-            v = &*a->value;
-            pos = *a->pos;
+            v = v2;
+            /* Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); */
+            /* v = &*a->value; */
+            /* pos = *a->pos; */
         }
 
         else {

src/libexpr/attr-set.hh

@@ -2,6 +2,7 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
+#include "value-cache.hh"
 
 #include <algorithm>
 #include <optional>
@@ -36,6 +37,7 @@ class Bindings
 public:
     typedef uint32_t size_t;
     Pos *pos;
+    ValueCache eval_cache;
 
 private:
     size_t size_, capacity_;

src/libexpr/context.cc

@@ -0,0 +1,21 @@
+#include "context.hh"
+
+namespace nix {
+
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+std::string encodeContext(std::string_view name, std::string_view path)
+{
+    return "!" + std::string(name) + "!" + std::string(path);
+}
+
+}

src/libexpr/context.hh

@@ -0,0 +1,11 @@
+#include "util.hh"
+
+namespace nix {
+
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s);
+
+std::string encodeContext(std::string_view name, std::string_view path);
+
+}

src/libexpr/eval-inline.hh

@@ -46,6 +46,12 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     }
     else if (v.isApp())
         callFunction(*v.app.left, *v.app.right, v, noPos);
+    else if (v.isCachedThunk()) {
+        auto evalCache = v.getEvalCache();
+        v.mkThunk(v.cachedThunk.thunk->env, v.cachedThunk.thunk->expr);
+        forceValue(v, pos);
+        v.setEvalCache(evalCache);
+    }
     else if (v.isBlackhole())
         throwEvalError(pos, "infinite recursion encountered");
 }

src/libexpr/eval.cc

@@ -1,4 +1,5 @@
 #include "eval.hh"
+#include "value-cache.hh"
 #include "hash.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -8,6 +9,7 @@
 #include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
+#include "value-cache.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -122,6 +124,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         str << "]";
         break;
     case tThunk:
+    case tCachedThunk:
     case tApp:
         str << "<CODE>";
         break;
@@ -193,7 +196,7 @@ string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tThunk: return "a thunk";
+        case tThunk: case tCachedThunk: return "a thunk";
         case tApp: return "a function application";
         case tBlackhole: return "a black hole";
     default:
@@ -216,7 +219,7 @@ bool Value::isTrivial() const
     return
         internalType != tApp
         && internalType != tPrimOpApp
-        && (internalType != tThunk
+        && ((internalType != tThunk && internalType != tCachedThunk)
             || (dynamic_cast<ExprAttrs *>(thunk.expr)
                 && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
             || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -442,6 +445,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
+    for (auto [_, cache] : evalCache) {
+        cache->commit();
+    }
 }
 
 
@@ -1107,68 +1113,235 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
     v = *v2;
 }
 
+unsigned long nrLookups = 0;
 
-static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
+std::pair<ValueCache::CacheResult, ValueCache> ValueCache::getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest)
 {
-    std::ostringstream out;
-    bool first = true;
-    for (auto & i : attrPath) {
-        if (!first) out << '.'; else first = false;
-        try {
-            out << getName(i, state, env);
-        } catch (Error & e) {
-            assert(!i.symbol.set());
-            out << "\"${" << *i.expr << "}\"";
-        }
-    }
-    return out.str();
+    if (!rawCache)
+        return { {NoCacheKey}, ValueCache(nullptr) };
+    auto resultingCursor = rawCache->findAlongAttrPath(selector);
+    if (!resultingCursor)
+        return { {CacheMiss}, ValueCache(nullptr) };
+
+    auto cachedValue = resultingCursor->getCachedValue();
+    auto cacheResult = std::visit(
+        overloaded{
+            [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
+            [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
+            [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
+            [&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
+            [&](tree_cache::missing_t x) {
+                return ValueCache::CacheResult{
+                    .returnCode = CacheHit,
+                    .lastQueriedSymbolIfMissing = x.attrName
+                };
+            },
+            [&](tree_cache::string_t s) {
+                PathSet context;
+                for (auto& [pathName, outputName] : s.second) {
+                    // If the cached value depends on some non-existent
+                    // path, we need to discard it and force the evaluation
+                    // to bring back the context in the store
+                    if (!state.store->isValidPath(
+                            state.store->parseStorePath(pathName)))
+                        return ValueCache::CacheResult{UnCacheable};
+                    context.insert("!" + outputName + "!" + pathName);
+                }
+                mkString(dest, s.first, context);
+                return ValueCache::CacheResult{CacheHit};
+            },
+            [&](tree_cache::wrapped_basetype<bool> b) {
+                dest.mkBool(b.value);
+                return ValueCache::CacheResult{CacheHit};
+            },
+            [&](tree_cache::wrapped_basetype<int64_t> i) {
+                dest.mkInt(i.value);
+                return ValueCache::CacheResult{CacheHit};
+            },
+            [&](tree_cache::wrapped_basetype<double> d) {
+                dest.mkFloat(d.value);
+                return ValueCache::CacheResult{CacheHit};
+            },
+        },
+        cachedValue);
+
+    return { cacheResult, ValueCache(resultingCursor) };
 }
 
+void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
+{
+    switch (cacheResult.returnCode) {
+        case ValueCache::CacheHit:
+            nrCacheHits++;
+            break;
+        case ValueCache::CacheMiss:
+            nrCacheMisses++;
+            break;
+        case ValueCache::UnCacheable:
+            nrUncacheable++;
+            break;
+        case ValueCache::NoCacheKey:
+            nrUncached++;
+            break;
+        case ValueCache::Forward:
+            nrCacheHits++;
+            break;
+    };
+}
 
-unsigned long nrLookups = 0;
+EvalState::LazyValueType EvalState::lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
+{
+    auto eval_cache = attrs.getEvalCache();
+    if (eval_cache.isEmpty()) {
+        forceValue(attrs, pos);
+        eval_cache = attrs.getEvalCache();
+    }
+    auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
+    updateCacheStats(cacheResult);
+
+    auto delayValue = [&](ValueCache & cursor) {
+            auto recordAsVar = new ExprCastedVar(&attrs);
+            auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
+            auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
+            thunk->expr = accessExpr;
+            thunk->env = &baseEnv;
+
+            dest.mkCachedThunk(
+                thunk,
+                new ValueCache(cursor)
+            );
+    };
+
+    switch (cacheResult.returnCode) {
+        case ValueCache::CacheHit:
+            if (cacheResult.lastQueriedSymbolIfMissing)
+                return LazyValueType::Missing;
+            return LazyValueType::PlainValue;
+        case ValueCache::UnCacheable:
+            delayValue(resultingCursor);
+            return LazyValueType::DelayedUncacheable;
+        case ValueCache::Forward:
+            delayValue(resultingCursor);
+            return LazyValueType::DelayedAttr;
+        default:
+            if(getAttrField(attrs, selector, pos, dest))
+                return LazyValueType::PlainValue;
+            else
+                return LazyValueType::Missing;
+    }
+
+}
+
+bool EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
+{
+    Pos * pos2 = 0;
+
+    auto eval_cache = attrs.getEvalCache();
+    if (eval_cache.isEmpty()) {
+        forceValue(attrs, pos);
+        eval_cache = attrs.getEvalCache();
+    }
+    auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
+    updateCacheStats(cacheResult);
+    switch (cacheResult.returnCode) {
+        case ValueCache::CacheHit:
+            if (cacheResult.lastQueriedSymbolIfMissing)
+                return false;
+            return true;
+        case ValueCache::CacheMiss:
+            resultingCursor = eval_cache;
+            break;
+        case ValueCache::Forward: // FIXME: Handle properly
+        case ValueCache::NoCacheKey:
+        case ValueCache::UnCacheable:
+            ;
+    }
+
+    forceValue(attrs, pos);
+
+    Value * vAttrs = &attrs;
+    try {
+        for (auto & name : selector) {
+            nrLookups++;
+            Bindings::iterator j;
+            if (vAttrs->type() != nAttrs ||
+                (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
+                return false;
+            }
+            vAttrs = j->value;
+            pos2 = j->pos;
+            try {
+                forceValue(*vAttrs, pos2 != NULL ? *pos2 : pos );
+            } catch (EvalError & e) {
+                resultingCursor.addFailedChild(name, e);
+                throw;
+            }
+            if (cacheResult.returnCode == ValueCache::CacheMiss) {
+                resultingCursor = resultingCursor.addChild(name, *vAttrs);
+                vAttrs->setEvalCache(resultingCursor);
+            }
+            if (countCalls && pos2) attrSelects[*pos2]++;
+        }
+
+    } catch (Error & e) {
+        if (pos2 && pos2->file != sDerivationNix) {
+            vector<string> strSelector;
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", concatStringsSep(".", selector));
+        }
+        throw;
+    }
+
+    if (cacheResult.returnCode == ValueCache::Forward)
+        vAttrs->setEvalCache(resultingCursor);
+
+    dest = *vAttrs;
+    return true;
+}
+
+void EvalState::getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
+{
+    if (!getAttrField(attrs, selector, pos, dest))
+        throw Error("Missing attribute path '%s'", "ImTooLazyToImplementThisRightNow");
+}
+
+std::vector<Symbol> EvalState::getFields(Value & attrs, const Pos & pos)
+{
+    auto eval_cache = attrs.getEvalCache();
+    if (eval_cache.isEmpty()) {
+        forceValue(attrs, pos);
+        eval_cache = attrs.getEvalCache();
+    }
+    if (auto attrNames = eval_cache.listChildren(symbols)) {
+        return *attrNames;
+    }
+
+    forceAttrs(attrs);
+    std::vector<Symbol> res;
+    for (auto & attr : *attrs.attrs)
+        res.push_back(attr.name);
+    return res;
+}
 
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
-    Pos * pos2 = 0;
-    Value * vAttrs = &vTmp;
 
     e->eval(state, env, vTmp);
 
-    try {
+    std::vector<Symbol> selector;
+    for (auto & i : attrPath)
+        selector.push_back(getName(i, state, env));
 
-        for (auto & i : attrPath) {
-            nrLookups++;
-            Bindings::iterator j;
-            Symbol name = getName(i, state, env);
-            if (def) {
-                state.forceValue(*vAttrs, pos);
-                if (vAttrs->type() != nAttrs ||
-                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                {
-                    def->eval(state, env, v);
-                    return;
-                }
-            } else {
-                state.forceAttrs(*vAttrs, pos);
-                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
-                    throwEvalError(pos, "attribute '%1%' missing", name);
-            }
-            vAttrs = j->value;
-            pos2 = j->pos;
-            if (state.countCalls && pos2) state.attrSelects[*pos2]++;
+    bool gotField = state.getAttrField(vTmp, selector, pos, v);
+    if (!gotField) {
+        if (def) {
+            def->eval(state, env, v);
+            return;
+        } else {
+            throwEvalError(pos, "Missing field");
         }
-
-        state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
-
-    } catch (Error & e) {
-        if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
-                showAttrPath(state, env, attrPath));
-        throw;
     }
 
-    v = *vAttrs;
 }
 
 
@@ -1351,9 +1524,38 @@ void EvalState::incrFunctionCall(ExprLambda * fun)
     functionCalls[fun]++;
 }
 
+std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
+{
+    if (!rawCache)
+        return std::nullopt;
+    return rawCache->getCachedValue();
+}
+
+std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
+{
+    auto ret = std::vector<Symbol>();
+    if (rawCache) {
+        auto cachedValue = rawCache->getCachedValue();
+            if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
+                for (auto & fieldStr : rawCache->getChildren())
+                    ret.push_back(symbols.create(fieldStr));
+            }
+            return ret;
+    }
+    return std::nullopt;
+}
 
 void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
+    if (auto evalCache = fun.getEvalCache(); !evalCache.isEmpty()) {
+        if (auto cacheValue = evalCache.getRawValue()) {
+            if (std::holds_alternative<tree_cache::attributeSet_t>(*cacheValue)) {
+                res = fun;
+                return;
+            }
+        }
+    }
+
     forceValue(fun);
 
     if (fun.type() == nAttrs) {
@@ -1699,18 +1901,6 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
-    } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
-}
-
-
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -1729,6 +1919,102 @@ std::vector<std::pair<Path, std::string>> Value::getContext()
     return res;
 }
 
+ValueCache & Value::getEvalCache()
+{
+    if (internalType == tAttrs) {
+        return attrs->eval_cache;
+    } else if (internalType == tCachedThunk) {
+        return *cachedThunk.cache;
+    } else {
+        return ValueCache::empty;
+    }
+}
+
+ValueCache ValueCache::empty = ValueCache(nullptr);
+
+tree_cache::AttrValue cachedValueFor(Value& v)
+{
+    tree_cache::AttrValue valueToCache;
+    switch (v.type()) {
+        case nThunk:
+            valueToCache = tree_cache::thunk_t{};
+            break;
+        case nNull:
+        case nList:
+        case nFunction:
+        case nExternal:
+            valueToCache = tree_cache::unknown_t{};
+            break;
+        case nBool:
+            valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
+            break;
+        case nString:
+            valueToCache = tree_cache::string_t{
+                v.string.s,
+                v.getContext()
+            };
+            break;
+        case nPath:
+            valueToCache = tree_cache::string_t{
+                v.path,
+                {}
+            };
+            break;
+        case nAttrs:
+            valueToCache = tree_cache::attributeSet_t{};
+            break;
+        case nInt:
+            valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
+            break;
+        case nFloat:
+            valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
+            break;
+    };
+    return valueToCache;
+}
+
+ValueCache ValueCache::addChild(const Symbol& name, Value& value)
+{
+    if (!rawCache)
+        return ValueCache::empty;
+
+    auto cachedValue = cachedValueFor(value);
+    auto ret = ValueCache(rawCache->addChild(name, cachedValue));
+    if (value.type() == nAttrs)
+        ret.addAttrSetChilds(*value.attrs);
+    return ret;
+}
+
+
+ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
+{
+    if (!rawCache) return ValueCache::empty;
+    return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
+}
+
+void ValueCache::addAttrSetChilds(Bindings & children)
+{
+    if (!rawCache) return;
+    for (auto & attr : children) {
+        // We could in theory directly store the value, but that would cause
+        // an infinite recursion in case of a cyclic attrset.
+        // So in a first time, just store a thunk
+        if (attr.value->type() == nAttrs)
+            rawCache->addChild(attr.name, tree_cache::thunk_t{});
+        else
+            addChild(attr.name, *attr.value);
+    }
+
+}
+
+void Value::setEvalCache(ValueCache & newCache)
+{
+    if (internalType == tAttrs) {
+        attrs->eval_cache = newCache;
+    } else if (internalType == tCachedThunk) {
+        cachedThunk.cache = &newCache;
+    }
+}
 
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
@@ -2011,6 +2297,13 @@ void EvalState::printStats()
         topObj.attr("nrLookups", nrLookups);
         topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
         topObj.attr("nrFunctionCalls", nrFunctionCalls);
+        {
+            auto cache = topObj.object("evalCache");
+            cache.attr("nrCacheMisses", nrCacheMisses);
+            cache.attr("nrCacheHits", nrCacheHits);
+            cache.attr("nrUncached", nrUncached);
+            cache.attr("nrUncacheable", nrUncacheable);
+        }
 #if HAVE_BOEHMGC
         {
             auto gc = topObj.object("gc");
@@ -2111,6 +2404,21 @@ Strings EvalSettings::getDefaultNixPath()
     return res;
 }
 
+std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
+{
+    if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
+        return iter->second;
+
+    if (!(evalSettings.useEvalCache && evalSettings.pureEval))
+        return nullptr;
+    auto thisCache = tree_cache::Cache::tryCreate(
+            cacheKey,
+            symbols
+    );
+    evalCache.insert({cacheKey, thisCache});
+    return thisCache;
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);

src/libexpr/eval.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "attr-set.hh"
+#include "context.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
@@ -115,6 +116,7 @@ private:
     typedef std::map<Path, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
+    std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
 
     SearchPath searchPath;
 
@@ -131,6 +133,8 @@ public:
     EvalState(const Strings & _searchPath, ref<Store> store);
     ~EvalState();
 
+    std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
+
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
@@ -310,6 +314,8 @@ public:
 
     void realiseContext(const PathSet & context);
 
+    void updateCacheStats(ValueCache::CacheResult);
+
 private:
 
     unsigned long nrEnvs = 0;
@@ -323,6 +329,10 @@ private:
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
+    unsigned long nrCacheHits = 0;
+    unsigned long nrCacheMisses = 0;
+    unsigned long nrUncacheable = 0;
+    unsigned long nrUncached = 0;
 
     bool countCalls;
 
@@ -342,6 +352,27 @@ private:
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+public:
+
+    bool getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
+
+    enum struct LazyValueType {
+        PlainValue,
+        DelayedUncacheable,
+        DelayedAttr,
+        Missing,
+    };
+
+    // Similar to `getAttrField`, but if the cache says that the result is an
+    // attribute set, just return a thunk to it rather than forcing it.
+    LazyValueType lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
+
+    // Similar to `getAttrField`, but throws an `Error` if the field can’t be
+    // found
+    void getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
+
+    std::vector<Symbol> getFields(Value & attrs, const Pos & pos);
 };
 
 
@@ -349,10 +380,6 @@ private:
 string showType(ValueType type);
 string showType(const Value & v);
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s);
-
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
 

src/libexpr/flake/flake.cc

@@ -596,30 +596,44 @@ void callFlake(EvalState & state,
     const LockedFlake & lockedFlake,
     Value & vRes)
 {
-    auto vLocks = state.allocValue();
+    auto lockExpr = new ExprString(
+        state.symbols.create(lockedFlake.lockFile.to_string())
+    );
+    auto subdirExpr = new ExprString(
+        state.symbols.create(lockedFlake.flake.lockedRef.subdir)
+    );
+
     auto vRootSrc = state.allocValue();
-    auto vRootSubdir = state.allocValue();
-    auto vTmp1 = state.allocValue();
-    auto vTmp2 = state.allocValue();
-
-    mkString(*vLocks, lockedFlake.lockFile.to_string());
-
     emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
 
-    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
-
-    static RootValue vCallFlake = nullptr;
-
-    if (!vCallFlake) {
-        vCallFlake = allocRootValue(state.allocValue());
-        state.eval(state.parseExprFromString(
+    static Expr * callFlakeExpr = nullptr;
+    if (!callFlakeExpr) {
+        callFlakeExpr = state.parseExprFromString(
             #include "call-flake.nix.gen.hh"
-            , "/"), **vCallFlake);
+            , "/");
     }
 
-    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
-    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
-    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
+    auto resExpr = new ExprApp(
+        new ExprApp(
+            new ExprApp(
+                callFlakeExpr,
+                lockExpr
+            ),
+            new ExprCastedVar(vRootSrc)
+        ),
+        subdirExpr
+    );
+    auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
+    thunk->expr = resExpr;
+    thunk->env = &state.baseEnv;
+    auto fingerprint = lockedFlake.getFingerprint();
+    auto treeCache = state.openTreeCache(fingerprint);
+    auto cacheRoot = treeCache ? treeCache->getRoot() : nullptr;
+    auto evalCache = new ValueCache(cacheRoot);
+    vRes.mkCachedThunk(
+        thunk,
+        evalCache
+    );
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)

src/libexpr/nixexpr.cc

@@ -440,6 +440,18 @@ string ExprLambda::showNamePos() const
     return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
 }
 
+void ExprCastedVar::show(std::ostream & str) const {
+    std::set<const Value*> active;
+    printValue(str, active, *v);
+}
+
+void ExprCastedVar::bindVars(const StaticEnv & env) {}
+void ExprCastedVar::eval(EvalState & state, Env & env, Value & v) {
+    v = std::move(*this->v);
+}
+Value * ExprCastedVar::maybeThunk(EvalState & state, Env & env) {
+    return v;
+}
 
 
 /* Symbol table. */

src/libexpr/nixexpr.hh

@@ -165,7 +165,12 @@ struct ExprSelect : Expr
     Expr * e, * def;
     AttrPath attrPath;
     ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
-    ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
+    ExprSelect(const Pos & pos, Expr * e, const Symbol & name)
+        : ExprSelect(pos, e, std::vector{name}) {};
+    ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & symbolicAttrPath) : pos(pos), e(e), def(0) {
+        for (auto & name : symbolicAttrPath)
+            attrPath.push_back(AttrName(name));
+    };
     COMMON_METHODS
 };
 
@@ -334,6 +339,14 @@ struct ExprPos : Expr
     COMMON_METHODS
 };
 
+struct ExprCastedVar : Expr
+{
+    Value * v;
+    ExprCastedVar(Value * v) : v(v) {};
+    Value * maybeThunk(EvalState & state, Env & env);
+    COMMON_METHODS
+};
+
 
 /* Static environments are used to map variable names onto (level,
    displacement) pairs used to obtain the value of the variable at

src/libexpr/tree-cache.cc

@@ -0,0 +1,392 @@
+#include "tree-cache.hh"
+#include "sqlite.hh"
+#include "store-api.hh"
+#include "context.hh"
+
+namespace nix::tree_cache {
+
+static const char * schema = R"sql(
+create table if not exists Attributes (
+    id          integer primary key autoincrement not null,
+    parent      integer not null,
+    name        text,
+    type        integer not null,
+    value       text,
+    context     text,
+    unique      (parent, name)
+);
+
+create index if not exists IndexByParent on Attributes(parent, name);
+)sql";
+
+struct AttrDb
+{
+    std::atomic_bool failed{false};
+
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertAttribute;
+        SQLiteStmt updateAttribute;
+        SQLiteStmt insertAttributeWithContext;
+        SQLiteStmt queryAttribute;
+        SQLiteStmt queryAttributes;
+        std::unique_ptr<SQLiteTxn> txn;
+    };
+
+    std::unique_ptr<Sync<State>> _state;
+
+    AttrDb(const Hash & fingerprint)
+        : _state(std::make_unique<Sync<State>>())
+    {
+        auto state(_state->lock());
+
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
+        createDirs(cacheDir);
+
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->insertAttribute.create(state->db,
+            "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
+
+        state->updateAttribute.create(state->db,
+            "update Attributes set type = ?, value = ?, context = ? where id = ?");
+
+        state->insertAttributeWithContext.create(state->db,
+            "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
+
+        state->queryAttribute.create(state->db,
+            "select id, type, value, context from Attributes where parent = ? and name = ?");
+
+        state->queryAttributes.create(state->db,
+            "select name, type from Attributes where parent = ?");
+
+        state->txn = std::make_unique<SQLiteTxn>(state->db);
+    }
+
+    ~AttrDb()
+    {
+        try {
+            auto state(_state->lock());
+            if (!failed)
+                state->txn->commit();
+            state->txn.reset();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    template<typename F>
+    AttrId doSQLite(F && fun)
+    {
+        if (failed) return 0;
+        try {
+            return fun();
+        } catch (SQLiteError &) {
+            ignoreException();
+            failed = true;
+            return 0;
+        }
+    }
+
+    /**
+     * Store a leaf of the tree in the db
+     */
+    AttrId addEntry(
+        const AttrKey & key,
+        const AttrValue & value)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+            auto rawValue = RawValue::fromVariant(value);
+
+            state->insertAttributeWithContext.use()
+                (key.first)
+                (key.second)
+                (rawValue.type)
+                (rawValue.value.value_or(""), rawValue.value.has_value())
+                (rawValue.serializeContext())
+                .exec();
+            AttrId rowId = state->db.getLastInsertedRowId();
+            assert(rowId);
+            return rowId;
+        });
+    }
+
+    std::optional<AttrId> getId(const AttrKey& key)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+        if (!queryAttribute.next()) return std::nullopt;
+
+        return (AttrType) queryAttribute.getInt(0);
+    }
+
+    AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
+    {
+        debug("cache: miss for the attribute %s", key.second);
+        if (auto existingId = getId(key)) {
+            setValue(*existingId, value);
+            return *existingId;
+        }
+        return addEntry(key, value);
+    }
+
+    void setValue(const AttrId & id, const AttrValue & value)
+    {
+        auto state(_state->lock());
+            auto rawValue = RawValue::fromVariant(value);
+
+            state->updateAttribute.use()
+                (rawValue.type)
+                (rawValue.value.value_or(""), rawValue.value.has_value())
+                (rawValue.serializeContext())
+                (id)
+                .exec();
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+        if (!queryAttribute.next()) return {};
+
+        auto rowId = (AttrType) queryAttribute.getInt(0);
+        auto type = (AttrType) queryAttribute.getInt(1);
+
+        switch (type) {
+            case AttrType::Attrs: {
+                return {{rowId, attributeSet_t()}};
+            }
+            case AttrType::String: {
+                std::vector<std::pair<Path, std::string>> context;
+                if (!queryAttribute.isNull(3))
+                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
+                        context.push_back(decodeContext(s));
+                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
+            }
+            case AttrType::Bool:
+                return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
+            case AttrType::Int:
+                return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
+            case AttrType::Double:
+                return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
+            case AttrType::Unknown:
+                return {{rowId, unknown_t{}}};
+            case AttrType::Thunk:
+                return {{rowId, thunk_t{}}};
+            case AttrType::Missing:
+                return {{rowId, missing_t{key.second}}};
+            case AttrType::Failed:
+                return {{rowId, failed_t{queryAttribute.getStr(2)}}};
+            default:
+                throw Error("unexpected type in evaluation cache");
+        }
+    }
+
+    std::vector<std::string> getChildren(AttrId parentId)
+    {
+        std::vector<std::string> res;
+        auto state(_state->lock());
+
+        auto queryAttributes(state->queryAttributes.use()(parentId));
+
+        while (queryAttributes.next()) {
+            if (queryAttributes.getInt(1) != AttrType::Missing)
+            res.push_back(queryAttributes.getStr(0));
+        }
+
+        return res;
+    }
+};
+
+Cache::Cache(const Hash & useCache,
+        SymbolTable & symbols)
+    : db(std::make_shared<AttrDb>(useCache))
+    , symbols(symbols)
+    , rootSymbol(symbols.create("%root%"))
+{
+}
+
+std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
+{
+    try {
+        return std::make_shared<Cache>(useCache, symbols);
+    } catch (SQLiteError &) {
+        ignoreException();
+        return nullptr;
+    }
+}
+
+void Cache::commit()
+{
+    if (db) {
+        debug("Saving the cache");
+        auto state(db->_state->lock());
+        if (state->txn->active) {
+            state->txn->commit();
+            state->txn.reset();
+            state->txn = std::make_unique<SQLiteTxn>(state->db);
+        }
+    }
+}
+
+Cursor::Ref Cache::getRoot()
+{
+    return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
+}
+
+Cursor::Cursor(
+    ref<Cache> root,
+    const Parent & parent,
+    const AttrValue& value
+    )
+    : root(root)
+    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
+    , label(parent ? parent->second : root->rootSymbol)
+    , cachedValue({root->db->setOrUpdate(getKey(), value), value})
+{
+}
+
+Cursor::Cursor(
+    ref<Cache> root,
+    const Parent & parent,
+    const AttrId & id,
+    const AttrValue & value
+    )
+    : root(root)
+    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
+    , label(parent ? parent->second : root->rootSymbol)
+    , cachedValue({id, value})
+{
+}
+
+
+AttrKey Cursor::getKey()
+{
+    if (!parentId)
+        return {0, root->rootSymbol};
+    return {*parentId, label};
+}
+
+AttrValue Cursor::getCachedValue()
+{
+    return cachedValue.second;
+}
+
+void Cursor::setValue(const AttrValue & v)
+{
+    root->db->setValue(cachedValue.first, v);
+    cachedValue.second = v;
+}
+
+Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
+{
+    Parent parent = {{*this, attrPath}};
+    auto childCursor = new Cursor(
+        root,
+        parent,
+        v
+    );
+    return childCursor;
+}
+
+std::vector<std::string> Cursor::getChildren()
+{
+    return root->db->getChildren(cachedValue.first);
+}
+
+std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
+{
+    auto cursorAtPath = findAlongAttrPath(attrPath);
+    if (cursorAtPath)
+        return cursorAtPath->getChildren();
+    return std::nullopt;
+}
+
+Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
+{
+    auto rawAttr = root->db->getValue({cachedValue.first, name});
+    if (rawAttr) {
+        Parent parent = {{*this, name}};
+        debug("cache: hit for the attribute %s", cachedValue.first);
+        return new Cursor (
+            root, parent, rawAttr->first,
+            rawAttr->second);
+    }
+    if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
+        // If the parent is an attribute set but we're not present in the db,
+        // then we're not a member of this attribute set. So mark as missing
+        return addChild(name, missing_t{name});
+    }
+    return nullptr;
+}
+
+Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+{
+    auto currentCursor = this;
+    for (auto & currentAccessor : attrPath) {
+        currentCursor = currentCursor->maybeGetAttr(currentAccessor);
+        if (!currentCursor)
+            break;
+        if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
+            break;
+        if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
+            break;
+    }
+    return currentCursor;
+}
+
+const RawValue RawValue::fromVariant(const AttrValue & value)
+{
+    RawValue res;
+    std::visit(overloaded{
+      [&](attributeSet_t x) { res.type = AttrType::Attrs; },
+      [&](string_t x) {
+        res.type = AttrType::String;
+        res.value = x.first;
+        res.context = x.second;
+      },
+      [&](wrapped_basetype<bool> x) {
+        res.type = AttrType::Bool;
+        res.value = x.value ? "1" : "0";
+      },
+      [&](wrapped_basetype<int64_t> x) {
+        res.type = AttrType::Int;
+        res.value = std::to_string(x.value);
+      },
+      [&](wrapped_basetype<double> x) {
+        res.type = AttrType::Double;
+        res.value = std::to_string(x.value);
+      },
+      [&](unknown_t x) { res.type = AttrType::Unknown; },
+      [&](missing_t x) { res.type = AttrType::Missing; },
+      [&](thunk_t x) { res.type = AttrType::Thunk; },
+      [&](failed_t x) {
+          res.type = AttrType::Failed;
+          res.value = x.error;
+      }
+    }, value);
+    return res;
+}
+
+std::string RawValue::serializeContext() const
+{
+    std::string res;
+    for (auto & elt : context) {
+        res.append(encodeContext(elt.second, elt.first));
+        res.push_back(' ');
+    }
+    if (!res.empty())
+        res.pop_back(); // Remove the trailing space
+    return res;
+}
+
+}

src/libexpr/tree-cache.hh

@@ -0,0 +1,156 @@
+/**
+ * caching for a tree-like data structure (like Nix values)
+ *
+ * The cache is an sqlite db whose rows are the nodes of the tree, with a
+ * pointer to their parent (except for the root of course)
+ */
+
+#pragma once
+
+#include "sync.hh"
+#include "hash.hh"
+#include "symbol-table.hh"
+
+#include <functional>
+#include <variant>
+
+namespace nix::tree_cache {
+
+struct AttrDb;
+class Cursor;
+
+class Cache : public std::enable_shared_from_this<Cache>
+{
+private:
+    friend class Cursor;
+
+    /**
+     * The database holding the cache
+     */
+    std::shared_ptr<AttrDb> db;
+
+    SymbolTable & symbols;
+
+    /**
+     * Distinguished symbol indicating the root of the tree
+     */
+    const Symbol rootSymbol;
+
+public:
+
+    Cache(
+        const Hash & useCache,
+        SymbolTable & symbols
+    );
+
+    static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
+
+    Cursor * getRoot();
+
+    /**
+     * Flush the cache to disk
+     */
+    void commit();
+};
+
+enum AttrType {
+    Unknown = 0,
+    Attrs = 1,
+    String = 2,
+    Bool = 3,
+    Int = 4,
+    Double = 5,
+    Thunk = 6,
+    Missing = 7, // Missing fields of attribute sets
+    Failed = 8,
+};
+
+struct attributeSet_t {};
+struct unknown_t {};
+struct thunk_t {};
+struct failed_t { string error; };
+struct missing_t { Symbol attrName; };
+
+// Putting several different primitive types in an `std::variant` partially
+// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
+// from one to another which breaks the exhaustiveness check.
+// So we wrap them in a trivial class just to force the disambiguation
+template<typename T>
+struct wrapped_basetype{ T value; };
+
+typedef uint64_t AttrId;
+
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+
+typedef std::variant<
+    attributeSet_t,
+    string_t,
+    unknown_t,
+    thunk_t,
+    missing_t,
+    failed_t,
+    wrapped_basetype<bool>,
+    wrapped_basetype<int64_t>,
+    wrapped_basetype<double>
+> AttrValue;
+
+struct RawValue {
+    AttrType type;
+    std::optional<std::string> value;
+    std::vector<std::pair<Path, std::string>> context;
+
+    std::string serializeContext() const;
+
+    static const RawValue fromVariant(const AttrValue&);
+    AttrValue toVariant() const;
+};
+
+/**
+ * View inside the cache.
+ *
+ * A `Cursor` represents a node in the cached tree (be it a leaf or not)
+ */
+class Cursor : public std::enable_shared_from_this<Cursor>
+{
+    /**
+     * The overall cache of which this cursor is a view
+     */
+    ref<Cache> root;
+
+    typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
+
+    std::optional<AttrId> parentId;
+    Symbol label;
+
+    std::pair<AttrId, AttrValue> cachedValue;
+
+    /**
+     * Get the identifier for this node in the database
+     */
+    AttrKey getKey();
+
+public:
+
+    using Ref = Cursor*;
+
+    // Create a new cache entry
+    Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
+    // Build a cursor from an existing cache entry
+    Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
+
+    AttrValue getCachedValue();
+
+    void setValue(const AttrValue & v);
+
+    Ref addChild(const Symbol & attrPath, const AttrValue & v);
+
+    Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
+    Ref maybeGetAttr(const Symbol & attrPath);
+
+
+    std::vector<std::string> getChildren();
+    std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
+};
+
+}

src/libexpr/value-cache.hh

@@ -0,0 +1,51 @@
+#pragma once
+#include "tree-cache.hh"
+
+namespace nix {
+struct Value;
+class EvalState;
+class Bindings;
+
+class ValueCache {
+    tree_cache::Cursor::Ref rawCache;
+
+public:
+
+    ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
+
+    static ValueCache empty;
+
+    bool isEmpty () { return rawCache == nullptr; }
+
+    enum ReturnCode {
+        // The cache result was an attribute set, so we forward it later in the
+        // chain
+        Forward,
+        CacheMiss,
+        CacheHit,
+        UnCacheable,
+        NoCacheKey,
+    };
+
+    struct CacheResult {
+        ReturnCode returnCode;
+
+        // In case the query returns a `missing_t`, the symbol that's missing
+        std::optional<Symbol> lastQueriedSymbolIfMissing;
+    };
+    std::pair<CacheResult, ValueCache> getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest);
+
+    ValueCache addChild(const Symbol & attrName, Value & value);
+    ValueCache addFailedChild(const Symbol & attrName, const Error & error);
+    ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
+    void addAttrSetChilds(Bindings & children);
+    void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
+
+    std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
+    std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
+
+    std::optional<tree_cache::AttrValue> getRawValue();
+
+    ValueCache() : rawCache(nullptr) {}
+};
+}

src/libexpr/value.hh

@@ -23,6 +23,7 @@ typedef enum {
     tApp,
     tLambda,
     tBlackhole,
+    tCachedThunk,
     tPrimOp,
     tPrimOpApp,
     tExternal,
@@ -56,6 +57,7 @@ struct Pos;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
+class ValueCache;
 
 
 typedef int64_t NixInt;
@@ -103,6 +105,10 @@ class ExternalValueBase
 
 std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 
+struct Thunk {
+    Env * env;
+    Expr * expr;
+};
 
 struct Value
 {
@@ -122,6 +128,7 @@ public:
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
     inline bool isBlackhole() const { return internalType == tBlackhole; };
+    inline bool isCachedThunk() const { return internalType == tCachedThunk; };
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
@@ -165,10 +172,13 @@ public:
             Value * * elems;
         } bigList;
         Value * smallList[2];
+        Thunk thunk;
         struct {
-            Env * env;
-            Expr * expr;
-        } thunk;
+            Thunk * thunk;
+            // This is a pointer only to prevent a recursive import as
+            // `EvalCache` is already a pointer so would fit very nicely here.
+            ValueCache * cache;
+        } cachedThunk;
         struct {
             Value * left, * right;
         } app;
@@ -199,7 +209,7 @@ public:
             case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
-            case tThunk: case tApp: case tBlackhole: return nThunk;
+            case tThunk: case tApp: case tBlackhole: case tCachedThunk: return nThunk;
         }
         abort();
     }
@@ -272,6 +282,13 @@ public:
         thunk.expr = ex;
     }
 
+    inline void mkCachedThunk(Thunk * t, ValueCache * cache)
+    {
+        internalType = tCachedThunk;
+        cachedThunk.thunk = t;
+        cachedThunk.cache = cache;
+    }
+
     inline void mkApp(Value * l, Value * r)
     {
         internalType = tApp;
@@ -349,6 +366,9 @@ public:
     bool isTrivial() const;
 
     std::vector<std::pair<Path, std::string>> getContext();
+
+    ValueCache & getEvalCache();
+    void setEvalCache(ValueCache &);
 };
 
 

src/libstore/build/derivation-goal.cc

@@ -739,63 +739,6 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 {
 }
 
-void runPostBuildHook(
-    Store & store,
-    Logger & logger,
-    const StorePath & drvPath,
-    StorePathSet outputPaths
-)
-{
-    auto hook = settings.postBuildHook;
-    if (hook == "")
-        return;
-
-    Activity act(logger, lvlInfo, actPostBuildHook,
-            fmt("running post-build-hook '%s'", settings.postBuildHook),
-            Logger::Fields{store.printStorePath(drvPath)});
-    PushActivity pact(act.id);
-    std::map<std::string, std::string> hookEnvironment = getEnv();
-
-    hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
-    hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
-
-    RunOptions opts(settings.postBuildHook, {});
-    opts.environment = hookEnvironment;
-
-    struct LogSink : Sink {
-        Activity & act;
-        std::string currentLine;
-
-        LogSink(Activity & act) : act(act) { }
-
-        void operator() (std::string_view data) override {
-            for (auto c : data) {
-                if (c == '\n') {
-                    flushLine();
-                } else {
-                    currentLine += c;
-                }
-            }
-        }
-
-        void flushLine() {
-            act.result(resPostBuildLogLine, currentLine);
-            currentLine.clear();
-        }
-
-        ~LogSink() {
-            if (currentLine != "") {
-                currentLine += '\n';
-                flushLine();
-            }
-        }
-    };
-    LogSink sink(act);
-
-    opts.standardOut = &sink;
-    opts.mergeStderrToStdout = true;
-    runProgram2(opts);
-}
 
 void DerivationGoal::buildDone()
 {
@@ -861,15 +804,57 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
-        StorePathSet outputPaths;
-        for (auto & [_, path] : finalOutputs)
-            outputPaths.insert(path);
-        runPostBuildHook(
-            worker.store,
-            *logger,
-            drvPath,
-            outputPaths
-        );
+        if (settings.postBuildHook != "") {
+            Activity act(*logger, lvlInfo, actPostBuildHook,
+                fmt("running post-build-hook '%s'", settings.postBuildHook),
+                Logger::Fields{worker.store.printStorePath(drvPath)});
+            PushActivity pact(act.id);
+            StorePathSet outputPaths;
+            for (auto i : drv->outputs) {
+                outputPaths.insert(finalOutputs.at(i.first));
+            }
+            std::map<std::string, std::string> hookEnvironment = getEnv();
+
+            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
+            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
+
+            RunOptions opts(settings.postBuildHook, {});
+            opts.environment = hookEnvironment;
+
+            struct LogSink : Sink {
+                Activity & act;
+                std::string currentLine;
+
+                LogSink(Activity & act) : act(act) { }
+
+                void operator() (std::string_view data) override {
+                    for (auto c : data) {
+                        if (c == '\n') {
+                            flushLine();
+                        } else {
+                            currentLine += c;
+                        }
+                    }
+                }
+
+                void flushLine() {
+                    act.result(resPostBuildLogLine, currentLine);
+                    currentLine.clear();
+                }
+
+                ~LogSink() {
+                    if (currentLine != "") {
+                        currentLine += '\n';
+                        flushLine();
+                    }
+                }
+            };
+            LogSink sink(act);
+
+            opts.standardOut = &sink;
+            opts.mergeStderrToStdout = true;
+            runProgram2(opts);
+        }
 
         if (buildMode == bmCheck) {
             cleanupPostOutputsRegisteredModeCheck();
@@ -925,8 +910,6 @@ void DerivationGoal::resolvedFinished() {
 
     auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
 
-    StorePathSet outputPaths;
-
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
@@ -944,10 +927,8 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
-            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
-            outputPaths.insert(realisation->outPath);
         } else {
             // If we don't have a realisation, then it must mean that something
             // failed when building the resolved drv
@@ -955,13 +936,6 @@ void DerivationGoal::resolvedFinished() {
         }
     }
 
-    runPostBuildHook(
-        worker.store,
-        *logger,
-        drvPath,
-        outputPaths
-    );
-
     // This is potentially a bit fishy in terms of error reporting. Not sure
     // how to do it in a cleaner way
     amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);

src/libstore/build/drv-output-substitution-goal.cc

@@ -17,13 +17,6 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
 void DrvOutputSubstitutionGoal::init()
 {
     trace("init");
-
-    /* If the derivation already exists, we’re done */
-    if (worker.store.queryRealisation(id)) {
-        amDone(ecSuccess);
-        return;
-    }
-
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
     tryNext();
 }
@@ -60,26 +53,6 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
-    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
-        if (depId != id) {
-            if (auto localOutputInfo = worker.store.queryRealisation(depId);
-                localOutputInfo && localOutputInfo->outPath != depPath) {
-                warn(
-                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
-                    "Local:  %s\n"
-                    "Remote: %s",
-                    sub->getUri(),
-                    depId.to_string(),
-                    worker.store.printStorePath(localOutputInfo->outPath),
-                    worker.store.printStorePath(depPath)
-                );
-                tryNext();
-                return;
-            }
-            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
-        }
-    }
-
     addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
     if (waitees.empty()) outPathValid();

src/libstore/build/local-derivation-goal.cc

@@ -292,7 +292,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         auto & localStore = getLocalStore();
         uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
         struct statvfs st;
-        if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
+        if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
             (uint64_t) st.f_bavail * st.f_bsize < required)
             diskFull = true;
         if (statvfs(tmpDir.c_str(), &st) == 0 &&
@@ -417,7 +417,7 @@ void LocalDerivationGoal::startBuilder()
     }
 
     auto & localStore = getLocalStore();
-    if (localStore.storeDir != localStore.realStoreDir.get()) {
+    if (localStore.storeDir != localStore.realStoreDir) {
         #if __linux__
             useChroot = true;
         #else
@@ -726,7 +726,7 @@ void LocalDerivationGoal::startBuilder()
     Path logFile = openLogFile();
 
     /* Create a pipe to get the output of the builder. */
-    builderOut.create();
+    //builderOut.create();
 
     builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
     if (!builderOut.readSide)
@@ -1333,18 +1333,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
     // XXX: This should probably be allowed if the realisation corresponds to
     // an allowed derivation
-    {
-        if (!goal.isAllowed(id))
-            throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
-        return next->queryRealisation(id);
-    }
+    { throw Error("queryRealisation"); }
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
     {
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
         StorePathSet newPaths;
-        std::set<Realisation> newRealisations;
 
         for (auto & req : paths) {
             if (!goal.isAllowed(req))
@@ -1357,28 +1352,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             auto p =  std::get_if<DerivedPath::Built>(&path);
             if (!p) continue;
             auto & bfd = *p;
-            auto drv = readDerivation(bfd.drvPath);
-            auto drvHashes = staticOutputHashes(*this, drv);
             auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
             for (auto & [outputName, outputPath] : outputs)
-                if (wantOutput(outputName, bfd.outputs)) {
+                if (wantOutput(outputName, bfd.outputs))
                     newPaths.insert(outputPath);
-                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-                        auto thisRealisation = next->queryRealisation(
-                            DrvOutput{drvHashes.at(outputName), outputName}
-                        );
-                        assert(thisRealisation);
-                        newRealisations.insert(*thisRealisation);
-                    }
-                }
         }
 
         StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
             goal.addDependency(path);
-        for (auto & real : Realisation::closure(*next, newRealisations))
-            goal.addedDrvOutputs.insert(real.id);
     }
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -2481,7 +2464,6 @@ void LocalDerivationGoal::registerOutputs()
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         PathLocks dynamicOutputLock;
-        dynamicOutputLock.setDeletion(true);
         auto optFixedPath = output.path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)
@@ -2505,7 +2487,6 @@ void LocalDerivationGoal::registerOutputs()
                 assert(newInfo.ca);
             } else {
                 auto destPath = worker.store.toRealPath(finalDestPath);
-                deletePath(destPath);
                 movePath(actualPath, destPath);
                 actualPath = destPath;
             }

src/libstore/build/local-derivation-goal.hh

@@ -108,9 +108,6 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Paths that were added via recursive Nix calls. */
     StorePathSet addedPaths;
 
-    /* Realisations that were added via recursive Nix calls. */
-    std::set<DrvOutput> addedDrvOutputs;
-
     /* Recursive Nix calls are only allowed to build or realize paths
        in the original input closure or added via a recursive Nix call
        (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@@ -119,11 +116,6 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
-    bool isAllowed(const DrvOutput & id)
-    {
-        return addedDrvOutputs.count(id);
-    }
-
     bool isAllowed(const DerivedPath & req);
 
     friend struct RestrictedStore;

src/libstore/ca-specific-schema.sql

@@ -3,19 +3,10 @@
 -- is enabled
 
 create table if not exists Realisations (
-    id integer primary key autoincrement not null,
     drvPath text not null,
     outputName text not null, -- symbolic output id, usually "out"
     outputPath integer not null,
     signatures text, -- space-separated list
+    primary key (drvPath, outputName),
     foreign key (outputPath) references ValidPaths(id) on delete cascade
 );
-
-create index if not exists IndexRealisations on Realisations(drvPath, outputName);
-
-create table if not exists RealisationsRefs (
-    referrer integer not null,
-    realisationReference integer,
-    foreign key (referrer) references Realisations(id) on delete cascade,
-    foreign key (realisationReference) references Realisations(id) on delete restrict
-);

src/libstore/daemon.cc

@@ -885,15 +885,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
     case wopRegisterDrvOutput: {
         logger->startWork();
-        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
-            auto outputId = DrvOutput::parse(readString(from));
-            auto outputPath = StorePath(readString(from));
-            store->registerDrvOutput(Realisation{
-                .id = outputId, .outPath = outputPath});
-        } else {
-            auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
-            store->registerDrvOutput(realisation);
-        }
+        auto outputId = DrvOutput::parse(readString(from));
+        auto outputPath = StorePath(readString(from));
+        store->registerDrvOutput(Realisation{
+            .id = outputId, .outPath = outputPath});
         logger->stopWork();
         break;
     }
@@ -903,15 +898,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         auto outputId = DrvOutput::parse(readString(from));
         auto info = store->queryRealisation(outputId);
         logger->stopWork();
-        if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
-            std::set<StorePath> outPaths;
-            if (info) outPaths.insert(info->outPath);
-            worker_proto::write(*store, to, outPaths);
-        } else {
-            std::set<Realisation> realisations;
-            if (info) realisations.insert(*info);
-            worker_proto::write(*store, to, realisations);
-        }
+        std::set<StorePath> outPaths;
+        if (info) outPaths.insert(info->outPath);
+        worker_proto::write(*store, to, outPaths);
         break;
     }
 

src/libstore/gc.cc

@@ -775,7 +775,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
         try {
 
-            AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
+            AutoCloseDir dir(opendir(realStoreDir.c_str()));
             if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
             /* Read the store and immediately delete all paths that
@@ -856,7 +856,7 @@ void LocalStore::autoGC(bool sync)
             return std::stoll(readFile(*fakeFreeSpaceFile));
 
         struct statvfs st;
-        if (statvfs(realStoreDir.get().c_str(), &st))
+        if (statvfs(realStoreDir.c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
 
         return (uint64_t) st.f_bavail * st.f_frsize;

src/libstore/local-fs-store.hh

@@ -18,9 +18,6 @@ struct LocalFSStoreConfig : virtual StoreConfig
     const PathSetting logDir{(StoreConfig*) this, false,
         rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
         "log", "directory where Nix will store state"};
-    const PathSetting realStoreDir{(StoreConfig*) this, false,
-        rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
-        "physical path to the Nix store"};
 };
 
 class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
@@ -37,7 +34,7 @@ public:
     /* Register a permanent GC root. */
     Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
 
-    virtual Path getRealStoreDir() { return realStoreDir; }
+    virtual Path getRealStoreDir() { return storeDir; }
 
     Path toRealPath(const Path & storePath) override
     {

src/libstore/local-store.cc

@@ -53,15 +53,12 @@ struct LocalStore::State::Stmts {
     SQLiteStmt InvalidatePath;
     SQLiteStmt AddDerivationOutput;
     SQLiteStmt RegisterRealisedOutput;
-    SQLiteStmt UpdateRealisedOutput;
     SQLiteStmt QueryValidDerivers;
     SQLiteStmt QueryDerivationOutputs;
     SQLiteStmt QueryRealisedOutput;
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
-    SQLiteStmt QueryRealisationReferences;
-    SQLiteStmt AddRealisationReference;
 };
 
 int getSchema(Path schemaPath)
@@ -79,7 +76,7 @@ int getSchema(Path schemaPath)
 
 void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 {
-    const int nixCASchemaVersion = 2;
+    const int nixCASchemaVersion = 1;
     int curCASchema = getSchema(schemaPath);
     if (curCASchema != nixCASchemaVersion) {
         if (curCASchema > nixCASchemaVersion) {
@@ -97,39 +94,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
               #include "ca-specific-schema.sql.gen.hh"
                 ;
             db.exec(schema);
-            curCASchema = nixCASchemaVersion;
         }
-
-        if (curCASchema < 2) {
-            SQLiteTxn txn(db);
-            // Ugly little sql dance to add a new `id` column and make it the primary key
-            db.exec(R"(
-                create table Realisations2 (
-                    id integer primary key autoincrement not null,
-                    drvPath text not null,
-                    outputName text not null, -- symbolic output id, usually "out"
-                    outputPath integer not null,
-                    signatures text, -- space-separated list
-                    foreign key (outputPath) references ValidPaths(id) on delete cascade
-                );
-                insert into Realisations2 (drvPath, outputName, outputPath, signatures)
-                    select drvPath, outputName, outputPath, signatures from Realisations;
-                drop table Realisations;
-                alter table Realisations2 rename to Realisations;
-            )");
-            db.exec(R"(
-                create index if not exists IndexRealisations on Realisations(drvPath, outputName);
-
-                create table if not exists RealisationsRefs (
-                    referrer integer not null,
-                    realisationReference integer,
-                    foreign key (referrer) references Realisations(id) on delete cascade,
-                    foreign key (realisationReference) references Realisations(id) on delete restrict
-                );
-            )");
-            txn.commit();
-        }
-
         writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
         lockFile(lockFd.get(), ltRead, true);
     }
@@ -141,6 +106,9 @@ LocalStore::LocalStore(const Params & params)
     , LocalStoreConfig(params)
     , Store(params)
     , LocalFSStore(params)
+    , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
+        "physical path to the Nix store"}
+    , realStoreDir(realStoreDir_)
     , dbDir(stateDir + "/db")
     , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
@@ -185,13 +153,13 @@ LocalStore::LocalStore(const Params & params)
             printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
         else {
             struct stat st;
-            if (stat(realStoreDir.get().c_str(), &st))
+            if (stat(realStoreDir.c_str(), &st))
                 throw SysError("getting attributes of path '%1%'", realStoreDir);
 
             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
-                if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
+                if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
                     throw SysError("changing ownership of path '%1%'", realStoreDir);
-                if (chmod(realStoreDir.get().c_str(), perm) == -1)
+                if (chmod(realStoreDir.c_str(), perm) == -1)
                     throw SysError("changing permissions on path '%1%'", realStoreDir);
             }
         }
@@ -346,18 +314,9 @@ LocalStore::LocalStore(const Params & params)
                 values (?, ?, (select id from ValidPaths where path = ?), ?)
                 ;
             )");
-        state->stmts->UpdateRealisedOutput.create(state->db,
-            R"(
-                update Realisations
-                    set signatures = ?
-                where
-                    drvPath = ? and
-                    outputName = ?
-                ;
-            )");
         state->stmts->QueryRealisedOutput.create(state->db,
             R"(
-                select Realisations.id, Output.path, Realisations.signatures from Realisations
+                select Output.path, Realisations.signatures from Realisations
                     inner join ValidPaths as Output on Output.id = Realisations.outputPath
                     where drvPath = ? and outputName = ?
                     ;
@@ -369,19 +328,6 @@ LocalStore::LocalStore(const Params & params)
                     where drvPath = ?
                     ;
             )");
-        state->stmts->QueryRealisationReferences.create(state->db,
-            R"(
-                select drvPath, outputName from Realisations
-                    join RealisationsRefs on realisationReference = Realisations.id
-                    where referrer = ?;
-            )");
-        state->stmts->AddRealisationReference.create(state->db,
-            R"(
-                insert or replace into RealisationsRefs (referrer, realisationReference)
-                values (
-                    ?,
-                    (select id from Realisations where drvPath = ? and outputName = ?));
-            )");
     }
 }
 
@@ -491,14 +437,14 @@ void LocalStore::makeStoreWritable()
     if (getuid() != 0) return;
     /* Check if /nix/store is on a read-only mount. */
     struct statvfs stat;
-    if (statvfs(realStoreDir.get().c_str(), &stat) != 0)
+    if (statvfs(realStoreDir.c_str(), &stat) != 0)
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
         if (unshare(CLONE_NEWNS) == -1)
             throw SysError("setting up a private mount namespace");
 
-        if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
+        if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
             throw SysError("remounting %1% writable", realStoreDir);
     }
 #endif
@@ -718,54 +664,14 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
     settings.requireExperimentalFeature("ca-derivations");
+    auto state(_state.lock());
     retrySQLite<void>([&]() {
-        auto state(_state.lock());
-        if (auto oldR = queryRealisation_(*state, info.id)) {
-            if (info.isCompatibleWith(*oldR)) {
-                auto combinedSignatures = oldR->signatures;
-                combinedSignatures.insert(info.signatures.begin(),
-                    info.signatures.end());
-                state->stmts->UpdateRealisedOutput.use()
-                    (concatStringsSep(" ", combinedSignatures))
-                    (info.id.strHash())
-                    (info.id.outputName)
-                    .exec();
-            } else {
-                throw Error("Trying to register a realisation of '%s', but we already "
-                            "have another one locally.\n"
-                            "Local:  %s\n"
-                            "Remote: %s",
-                    info.id.to_string(),
-                    printStorePath(oldR->outPath),
-                    printStorePath(info.outPath)
-                );
-            }
-        } else {
-            state->stmts->RegisterRealisedOutput.use()
-                (info.id.strHash())
-                (info.id.outputName)
-                (printStorePath(info.outPath))
-                (concatStringsSep(" ", info.signatures))
-                .exec();
-        }
-        uint64_t myId = state->db.getLastInsertedRowId();
-        for (auto & [outputId, depPath] : info.dependentRealisations) {
-            auto localRealisation = queryRealisationCore_(*state, outputId);
-            if (!localRealisation)
-                throw Error("unable to register the derivation '%s' as it "
-                            "depends on the non existent '%s'",
-                    info.id.to_string(), outputId.to_string());
-            if (localRealisation->second.outPath != depPath)
-                throw Error("unable to register the derivation '%s' as it "
-                            "depends on a realisation of '%s' that doesn’t"
-                            "match what we have locally",
-                    info.id.to_string(), outputId.to_string());
-            state->stmts->AddRealisationReference.use()
-                (myId)
-                (outputId.strHash())
-                (outputId.outputName)
-                .exec();
-        }
+        state->stmts->RegisterRealisedOutput.use()
+            (info.id.strHash())
+            (info.id.outputName)
+            (printStorePath(info.outPath))
+            (concatStringsSep(" ", info.signatures))
+            .exec();
     });
 }
 
@@ -1776,69 +1682,23 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
     }
 }
 
-std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_(
-        LocalStore::State & state,
-        const DrvOutput & id)
-{
-    auto useQueryRealisedOutput(
-            state.stmts->QueryRealisedOutput.use()
-                (id.strHash())
-                (id.outputName));
-    if (!useQueryRealisedOutput.next())
-        return std::nullopt;
-    auto realisationDbId = useQueryRealisedOutput.getInt(0);
-    auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
-    auto signatures =
-        tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
-
-    return {{
-        realisationDbId,
-        Realisation{
-            .id = id,
-            .outPath = outputPath,
-            .signatures = signatures,
-        }
-    }};
-}
-
-std::optional<const Realisation> LocalStore::queryRealisation_(
-            LocalStore::State & state,
-            const DrvOutput & id)
-{
-    auto maybeCore = queryRealisationCore_(state, id);
-    if (!maybeCore)
-        return std::nullopt;
-    auto [realisationDbId, res] = *maybeCore;
-
-    std::map<DrvOutput, StorePath> dependentRealisations;
-    auto useRealisationRefs(
-        state.stmts->QueryRealisationReferences.use()
-            (realisationDbId));
-    while (useRealisationRefs.next()) {
-        auto depId = DrvOutput {
-            Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
-            useRealisationRefs.getStr(1),
-        };
-        auto dependentRealisation = queryRealisationCore_(state, depId);
-        assert(dependentRealisation); // Enforced by the db schema
-        auto outputPath = dependentRealisation->second.outPath;
-        dependentRealisations.insert({depId, outputPath});
-    }
-
-    res.dependentRealisations = dependentRealisations;
-
-    return { res };
-}
-
-std::optional<const Realisation>
-LocalStore::queryRealisation(const DrvOutput & id)
-{
-    return retrySQLite<std::optional<const Realisation>>([&]() {
+std::optional<const Realisation> LocalStore::queryRealisation(
+    const DrvOutput& id) {
+    typedef std::optional<const Realisation> Ret;
+    return retrySQLite<Ret>([&]() -> Ret {
         auto state(_state.lock());
-        return queryRealisation_(*state, id);
+        auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
+            id.outputName));
+        if (!use.next())
+            return std::nullopt;
+        auto outputPath = parseStorePath(use.getStr(0));
+        auto signatures = tokenizeString<StringSet>(use.getStr(1));
+        return Ret{Realisation{
+            .id = id, .outPath = outputPath, .signatures = signatures}};
     });
 }
 
+
 FixedOutputHash LocalStore::hashCAPath(
     const FileIngestionMethod & method, const HashType & hashType,
     const StorePath & path)

src/libstore/local-store.hh

@@ -83,6 +83,9 @@ private:
 
 public:
 
+    PathSetting realStoreDir_;
+
+    const Path realStoreDir;
     const Path dbDir;
     const Path linksDir;
     const Path reservedPath;
@@ -203,8 +206,6 @@ public:
     void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
     void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
 
-    std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
-    std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
     std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
 
 private:
@@ -278,6 +279,8 @@ private:
     void signPathInfo(ValidPathInfo & info);
     void signRealisation(Realisation &);
 
+    Path getRealStoreDir() override { return realStoreDir; }
+
     void createUser(const std::string & userName, uid_t userId) override;
 
     // XXX: Make a generic `Store` method

src/libstore/misc.cc

@@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(i);
 
             if (includeDerivers && path.isDerivation())
-                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
-                    if (maybeOutPath && isValidPath(*maybeOutPath))
-                        res.insert(*maybeOutPath);
+                for (auto& i : queryDerivationOutputs(path))
+                    if (isValidPath(i) && queryPathInfo(i)->deriver == path)
+                        res.insert(i);
             return res;
         };
     else
@@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                     res.insert(ref);
 
             if (includeOutputs && path.isDerivation())
-                for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
-                    if (maybeOutPath && isValidPath(*maybeOutPath))
-                        res.insert(*maybeOutPath);
+                for (auto& i : queryDerivationOutputs(path))
+                    if (isValidPath(i))
+                        res.insert(i);
 
             if (includeDerivers && info->deriver && isValidPath(*info->deriver))
                 res.insert(*info->deriver);
@@ -254,44 +254,5 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         }});
 }
 
-std::map<DrvOutput, StorePath> drvOutputReferences(
-    const std::set<Realisation> & inputRealisations,
-    const StorePathSet & pathReferences)
-{
-    std::map<DrvOutput, StorePath> res;
 
-    for (const auto & input : inputRealisations) {
-        if (pathReferences.count(input.outPath)) {
-            res.insert({input.id, input.outPath});
-        }
-    }
-
-    return res;
-}
-
-std::map<DrvOutput, StorePath> drvOutputReferences(
-    Store & store,
-    const Derivation & drv,
-    const StorePath & outputPath)
-{
-    std::set<Realisation> inputRealisations;
-
-    for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
-        auto outputHashes =
-            staticOutputHashes(store, store.readDerivation(inputDrv));
-        for (const auto& outputName : outputNames) {
-            auto thisRealisation = store.queryRealisation(
-                DrvOutput{outputHashes.at(outputName), outputName});
-            if (!thisRealisation)
-                throw Error(
-                    "output '%s' of derivation '%s' isn’t built", outputName,
-                    store.printStorePath(inputDrv));
-            inputRealisations.insert(*thisRealisation);
-        }
-    }
-
-    auto info = store.queryPathInfo(outputPath);
-
-    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
-}
 }

src/libstore/optimise-store.cc

@@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
        permissions). */
-    bool mustToggle = dirOf(path) != realStoreDir.get();
+    bool mustToggle = dirOf(path) != realStoreDir;
     if (mustToggle) makeWritable(dirOf(path));
 
     /* When we're done, make the directory read-only again and reset

src/libstore/parsed-derivations.cc

@@ -91,8 +91,6 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
     StringSet res;
     for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
         res.insert(i);
-    if (!derivationHasKnownOutputPaths(drv.type()))
-        res.insert("ca-derivations");
     return res;
 }
 

src/libstore/realisation.cc

@@ -1,6 +1,5 @@
 #include "realisation.hh"
 #include "store-api.hh"
-#include "closure.hh"
 #include <nlohmann/json.hpp>
 
 namespace nix {
@@ -22,52 +21,11 @@ std::string DrvOutput::to_string() const {
     return strHash() + "!" + outputName;
 }
 
-std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
-{
-    std::set<Realisation> res;
-    Realisation::closure(store, startOutputs, res);
-    return res;
-}
-
-void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
-{
-    auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
-        std::set<Realisation> res;
-        for (auto& [currentDep, _] : current.dependentRealisations) {
-            if (auto currentRealisation = store.queryRealisation(currentDep))
-                res.insert(*currentRealisation);
-            else
-                throw Error(
-                    "Unrealised derivation '%s'", currentDep.to_string());
-        }
-        return res;
-    };
-
-    computeClosure<Realisation>(
-        startOutputs, res,
-        [&](const Realisation& current,
-            std::function<void(std::promise<std::set<Realisation>>&)>
-                processEdges) {
-            std::promise<std::set<Realisation>> promise;
-            try {
-                auto res = getDeps(current);
-                promise.set_value(res);
-            } catch (...) {
-                promise.set_exception(std::current_exception());
-            }
-            return processEdges(promise);
-        });
-}
-
 nlohmann::json Realisation::toJSON() const {
-    auto jsonDependentRealisations = nlohmann::json::object();
-    for (auto & [depId, depOutPath] : dependentRealisations)
-        jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
     return nlohmann::json{
         {"id", id.to_string()},
         {"outPath", outPath.to_string()},
         {"signatures", signatures},
-        {"dependentRealisations", jsonDependentRealisations},
     };
 }
 
@@ -93,16 +51,10 @@ Realisation Realisation::fromJSON(
     if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
         signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
 
-    std::map <DrvOutput, StorePath> dependentRealisations;
-    if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end())
-        for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>())
-            dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
-
     return Realisation{
         .id = DrvOutput::parse(getField("id")),
         .outPath = StorePath(getField("outPath")),
         .signatures = signatures,
-        .dependentRealisations = dependentRealisations,
     };
 }
 
@@ -140,16 +92,6 @@ StorePath RealisedPath::path() const {
     return std::visit([](auto && arg) { return arg.getPath(); }, raw);
 }
 
-bool Realisation::isCompatibleWith(const Realisation & other) const
-{
-    assert (id == other.id);
-    if (outPath == other.outPath) {
-        assert(dependentRealisations == other.dependentRealisations);
-        return true;
-    }
-    return false;
-}
-
 void RealisedPath::closure(
     Store& store,
     const RealisedPath::Set& startPaths,

src/libstore/realisation.hh

@@ -28,14 +28,6 @@ struct Realisation {
 
     StringSet signatures;
 
-    /**
-     * The realisations that are required for the current one to be valid.
-     *
-     * When importing this realisation, the store will first check that all its
-     * dependencies exist, and map to the correct output path
-     */
-    std::map<DrvOutput, StorePath> dependentRealisations;
-
     nlohmann::json toJSON() const;
     static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
 
@@ -44,11 +36,6 @@ struct Realisation {
     bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
     size_t checkSignatures(const PublicKeys & publicKeys) const;
 
-    static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
-    static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
-
-    bool isCompatibleWith(const Realisation & other) const;
-
     StorePath getPath() const { return outPath; }
 
     GENERATE_CMP(Realisation, me->id, me->outPath);

src/libstore/remote-store.cc

@@ -653,12 +653,8 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
 {
     auto conn(getConnection());
     conn->to << wopRegisterDrvOutput;
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
-        conn->to << info.id.to_string();
-        conn->to << std::string(info.outPath.to_string());
-    } else {
-        worker_proto::write(*this, conn->to, info);
-    }
+    conn->to << info.id.to_string();
+    conn->to << std::string(info.outPath.to_string());
     conn.processStderr();
 }
 
@@ -668,17 +664,10 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
-        auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
-        if (outPaths.empty())
-            return std::nullopt;
-        return {Realisation{.id = id, .outPath = *outPaths.begin()}};
-    } else {
-        auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
-        if (realisations.empty())
-            return std::nullopt;
-        return *realisations.begin();
-    }
+    auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
+    if (outPaths.empty())
+        return std::nullopt;
+    return {Realisation{.id = id, .outPath = *outPaths.begin()}};
 }
 
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)

src/libstore/store-api.cc

@@ -337,13 +337,6 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     return info;
 }
 
-StringSet StoreConfig::getDefaultSystemFeatures()
-{
-    auto res = settings.systemFeatures.get();
-    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
-        res.insert("ca-derivations");
-    return res;
-}
 
 Store::Store(const Params & params)
     : StoreConfig(params)
@@ -787,39 +780,20 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
     RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
 {
     StorePathSet storePaths;
-    std::set<Realisation> toplevelRealisations;
+    std::set<Realisation> realisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto realisation = std::get_if<Realisation>(&path.raw)) {
             settings.requireExperimentalFeature("ca-derivations");
-            toplevelRealisations.insert(*realisation);
+            realisations.insert(*realisation);
         }
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
-
-    ThreadPool pool;
-
     try {
-        // Copy the realisation closure
-        processGraph<Realisation>(
-            pool, Realisation::closure(*srcStore, toplevelRealisations),
-            [&](const Realisation& current) -> std::set<Realisation> {
-                std::set<Realisation> children;
-                for (const auto& [drvOutput, _] : current.dependentRealisations) {
-                    auto currentChild = srcStore->queryRealisation(drvOutput);
-                    if (!currentChild)
-                        throw Error(
-                            "Incomplete realisation closure: '%s' is a "
-                            "dependency of '%s' but isn’t registered",
-                            drvOutput.to_string(), current.id.to_string());
-                    children.insert(*currentChild);
-                }
-                return children;
-            },
-            [&](const Realisation& current) -> void {
-                dstStore->registerDrvOutput(current, checkSigs);
-            });
-    } catch (MissingExperimentalFeature& e) {
+        for (auto & realisation : realisations) {
+            dstStore->registerDrvOutput(realisation, checkSigs);
+        }
+    } catch (MissingExperimentalFeature & e) {
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
         // to at least copy the output paths.

src/libstore/store-api.hh

@@ -180,8 +180,6 @@ struct StoreConfig : public Config
 
     StoreConfig() = delete;
 
-    StringSet getDefaultSystemFeatures();
-
     virtual ~StoreConfig() { }
 
     virtual const std::string name() = 0;
@@ -198,7 +196,7 @@ struct StoreConfig : public Config
 
     Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
 
-    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
+    Setting<StringSet> systemFeatures{this, settings.systemFeatures,
         "system-features",
         "Optional features that the system this store builds on implements (like \"kvm\")."};
 
@@ -866,9 +864,4 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
 
 std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
 
-std::map<DrvOutput, StorePath> drvOutputReferences(
-    Store & store,
-    const Derivation & drv,
-    const StorePath & outputPath);
-
 }

src/libstore/worker-protocol.hh

@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 31)
+#define PROTOCOL_VERSION (1 << 8 | 30)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

src/libutil/comparator.hh

@@ -25,8 +25,6 @@
     }
 #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
 #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
-#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
     GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args) \
-    GENERATE_NEQ(args)
+    GENERATE_LEQ(args)

src/nix-build/nix-build.cc

@@ -387,12 +387,6 @@ static void main_nix_build(int argc, char * * argv)
 
         if (dryRun) return;
 
-        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-            auto resolvedDrv = drv.tryResolve(*store);
-            assert(resolvedDrv && "Successfully resolved the derivation");
-            drv = *resolvedDrv;
-        }
-
         // Set the environment.
         auto env = getEnv();
 

src/nix/build.cc

@@ -69,7 +69,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                             store2->addPermRoot(bo.path, absPath(symlink));
                         },
                         [&](BuiltPath::Built bfd) {
-                            for (auto & output : bfd.outputs) {
+                            auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
+                            for (auto & output : builtOutputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);

src/nix/develop.cc

@@ -144,26 +144,17 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
+    for (auto & output : drv.outputs) {
+        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+        drv.env[output.first] = "";
+    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-        for (auto & output : drv.outputs) {
-            output.second = {
-                .output = DerivationOutputDeferred{},
-            };
-            drv.env[output.first] = hashPlaceholder(output.first);
-        }
-    } else {
-        for (auto & output : drv.outputs) {
-            output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
-            drv.env[output.first] = "";
-        }
-        Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
+    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
 
-        for (auto & output : drv.outputs) {
-            auto outPath = store->makeOutputPath(output.first, h, drv.name);
-            output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
-            drv.env[output.first] = store->printStorePath(outPath);
-        }
+    for (auto & output : drv.outputs) {
+        auto outPath = store->makeOutputPath(output.first, h, drv.name);
+        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+        drv.env[output.first] = store->printStorePath(outPath);
     }
 
     auto shellDrvPath = writeDerivation(*store, drv);
@@ -171,7 +162,8 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Build the derivation. */
     store->buildPaths({DerivedPath::Built{shellDrvPath}});
 
-    for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) {
+    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
+        auto & [_1, optPath] = outputAndOptPath;
         assert(optPath);
         auto & outPath = *optPath;
         assert(store->isValidPath(outPath));
@@ -193,7 +185,6 @@ struct Common : InstallableCommand, MixProfile
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",
-        "NIX_REMOTE",
         "PPID",
         "PWD",
         "SHELLOPTS",

src/nix/profile-remove.md

@@ -15,7 +15,6 @@ R""(
   ```
 
 * Remove all packages:
-
   ```console
   # nix profile remove '.*'
   ```

src/nix/run.cc

@@ -43,8 +43,8 @@ struct RunCommon : virtual Command
            helper program (chrootHelper() below) to do the work. */
         auto store2 = store.dynamic_pointer_cast<LocalStore>();
 
-        if (store2 && store->storeDir != store2->getRealStoreDir()) {
-            Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
+        if (store2 && store->storeDir != store2->realStoreDir) {
+            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
             for (auto & arg : args) helperArgs.push_back(arg);
 
             execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());

src/nix/search.cc

@@ -9,6 +9,7 @@
 #include "shared.hh"
 #include "eval-cache.hh"
 #include "attr-path.hh"
+#include "value.hh"
 
 #include <regex>
 #include <fstream>
@@ -81,31 +82,46 @@ struct CmdSearch : InstallableCommand, MixJSON
 
         uint64_t results = 0;
 
-        std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
+        std::function<void(Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
 
-        visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)
+        Value * vTmp = state->allocValue();
+
+        visit = [&](Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
-            try {
-                auto recurse = [&]()
-                {
-                    for (const auto & attr : cursor.getAttrs()) {
-                        auto cursor2 = cursor.getAttr(attr);
-                        auto attrPath2(attrPath);
-                        attrPath2.push_back(attr);
-                        visit(*cursor2, attrPath2, false);
-                    }
-                };
+            auto recurse = [&]()
+            {
+                for (auto & attrName : state->getFields(current, noPos)) {
+                  try {
+                    auto attrPath2(attrPath);
+                    attrPath2.push_back(attrName);
+                    auto attrValue = state->allocValue();
+                    auto value_ = allocRootValue(attrValue);
+                    auto childField = state->lazyGetAttrField(current, {attrName}, noPos,
+                                            *attrValue);
+                    if (childField != EvalState::LazyValueType::DelayedUncacheable)
+                        visit(*attrValue, attrPath2, false);
+                  } catch (EvalError &e) {
+                    if (!(attrPath.size() > 0 &&
+                          attrPath[0] == "legacyPackages"))
+                      throw;
+                  }
+                }
+            };
 
-                if (cursor.isDerivation()) {
+            try {
+                auto maybeTypeField = state->lazyGetAttrField(current, {state->sType}, noPos, *vTmp);
+                if (maybeTypeField == EvalState::LazyValueType::PlainValue
+                    && vTmp->type() == nix::nString
+                    && strcmp(vTmp->string.s, "derivation") == 0) {
                     size_t found = 0;
 
-                    DrvName name(cursor.getAttr("name")->getString());
+                    state->getAttrFieldThrow(current, {state->sName}, noPos, *vTmp);
+                    DrvName name(state->forceString(*vTmp));
 
-                    auto aMeta = cursor.maybeGetAttr("meta");
-                    auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
-                    auto description = aDescription ? aDescription->getString() : "";
+                    auto hasDescription = state->getAttrField(current, {state->sMeta, state->sDescription}, noPos, *vTmp);
+                    auto description = hasDescription ? state->forceString(*vTmp) : "";
                     std::replace(description.begin(), description.end(), '\n', ' ');
                     auto attrPath2 = concatStringsSep(".", attrPath);
 
@@ -118,8 +134,8 @@ struct CmdSearch : InstallableCommand, MixJSON
                         std::regex_search(name.name, nameMatch, regex);
                         std::regex_search(description, descriptionMatch, regex);
                         if (!attrPathMatch.empty()
-                            || !nameMatch.empty()
-                            || !descriptionMatch.empty())
+                                || !nameMatch.empty()
+                                || !descriptionMatch.empty())
                             found++;
                     }
 
@@ -134,39 +150,37 @@ struct CmdSearch : InstallableCommand, MixJSON
                             auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
                             if (results > 1) logger->cout("");
                             logger->cout(
-                                "* %s%s",
-                                wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
-                                name.version != "" ? " (" + name.version + ")" : "");
+                                    "* %s%s",
+                                    wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
+                                    name.version != "" ? " (" + name.version + ")" : "");
                             if (description != "")
                                 logger->cout(
-                                    "  %s", hilite(description, descriptionMatch, ANSI_NORMAL));
+                                        "  %s", hilite(description, descriptionMatch, ANSI_NORMAL));
                         }
                     }
-                }
-
-                else if (
-                    attrPath.size() == 0
-                    || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
-                    || (attrPath[0] == "packages" && attrPath.size() <= 2))
+                } else if (
+                        attrPath.size() == 0
+                        || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
+                        || (attrPath[0] == "packages" && attrPath.size() <= 2))
                     recurse();
 
                 else if (initialRecurse)
                     recurse();
 
                 else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
-                    auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
-                    if (attr && attr->getBool())
+                    auto recurseFieldInfo = state->lazyGetAttrField(current, {state->sRecurseForDerivations}, noPos, *vTmp);
+                    auto hasRecurse = recurseFieldInfo == EvalState::LazyValueType::PlainValue;
+                    if (hasRecurse && state->forceBool(*vTmp, noPos))
                         recurse();
                 }
-
             } catch (EvalError & e) {
                 if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
                     throw;
             }
         };
 
-        for (auto & [cursor, prefix] : installable->getCursors(*state))
-            visit(*cursor, parseAttrPath(*state, prefix), true);
+        for (auto & [value, pos, prefix] : installable->toValues(*state))
+            visit(*value, parseAttrPath(*state, prefix), true);
 
         if (!json && !results)
             throw Error("no results for the given search term(s)!");

tests/build-remote-content-addressed-floating.sh

@@ -4,6 +4,4 @@ file=build-hook-ca-floating.nix
 
 sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
 
-CONTENT_ADDRESSED=true
-
 source build-remote.sh

tests/build-remote.sh

@@ -6,17 +6,12 @@ unset NIX_STATE_DIR
 
 function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
 
-EXTRA_SYSTEM_FEATURES=()
-if [[ -n "$CONTENT_ADDRESSED" ]]; then
-    EXTRA_SYSTEM_FEATURES=("ca-derivations")
-fi
-
 builders=(
   # system-features will automatically be added to the outer URL, but not inner
   # remote-store URL.
-  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})"
-  "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})"
-  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})"
+  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
+  "$TEST_ROOT/machine2 - - 1 1 bar"
+  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
 )
 
 chmod -R +w $TEST_ROOT/machine* || true

tests/build.sh

@@ -1,7 +1,7 @@
 source common.sh
 
 expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
-nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
+nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
     (.outputs.first | match(".*multiple-outputs-a-first")) and
@@ -10,10 +10,10 @@ nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-statu
     (.drvPath | match(".*multiple-outputs-b.drv")) and
     (.outputs.out | match(".*multiple-outputs-b")))
 '
+
 testNormalization () {
     clearStore
-    outPath=$(nix-build ./simple.nix --no-out-link)
+    outPath=$(nix-build ./simple.nix)
     test "$(stat -c %Y $outPath)" -eq 1
 }
-
 testNormalization

tests/ca/build-with-garbage-path.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# Regression test for https://github.com/NixOS/nix/issues/4858
-
-source common.sh
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
-
-# Get the output path of `rootCA`, and put some garbage instead
-outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"
-nix-store --delete "$outPath"
-touch "$outPath"
-
-# The build should correctly remove the garbage and put the expected path instead
-nix-build ./content-addressed.nix -A rootCA --no-out-link
-
-# Rebuild it. This shouldn’t overwrite the existing path
-oldInode=$(stat -c '%i' "$outPath")
-nix-build ./content-addressed.nix -A rootCA --no-out-link --arg seed 2
-newInode=$(stat -c '%i' "$outPath")
-[[ "$oldInode" == "$newInode" ]]

tests/ca/duplicate-realisation-in-closure.sh

@@ -1,26 +0,0 @@
-source ./common.sh
-
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
-
-export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
-export REMOTE_STORE="file://$REMOTE_STORE_DIR"
-
-rm -rf $REMOTE_STORE_DIR
-clearStore
-
-# Build dep1 and push that to the binary cache.
-# This entails building (and pushing) current-time.
-nix copy --to "$REMOTE_STORE" -f nondeterministic.nix dep1
-clearStore
-sleep 2 # To make sure that `$(date)` will be different
-# Build dep2.
-# As we’ve cleared the cache, we’ll have to rebuild current-time. And because
-# the current time isn’t the same as before, this will yield a new (different)
-# realisation
-nix build -f nondeterministic.nix dep2 --no-link
-
-# Build something that depends both on dep1 and dep2.
-# If everything goes right, we should rebuild dep2 rather than fetch it from
-# the cache (because that would mean duplicating `current-time` in the closure),
-# and have `dep1 == dep2`.
-nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link

tests/ca/gc.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-# Ensure that garbage collection works properly with ca derivations
-
-source common.sh
-
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
-
-export NIX_TESTS_CA_BY_DEFAULT=1
-
-cd ..
-source gc.sh

tests/ca/nix-shell.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-source common.sh
-
-sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
-
-CONTENT_ADDRESSED=true
-cd ..
-source ./nix-shell.sh
-

tests/ca/nondeterministic.nix

@@ -1,35 +0,0 @@
-with import ./config.nix;
-
-let mkCADerivation = args: mkDerivation ({
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
-} // args);
-in
-
-rec {
-  currentTime = mkCADerivation {
-    name = "current-time";
-    buildCommand = ''
-      mkdir $out
-      echo $(date) > $out/current-time
-    '';
-  };
-  dep = seed: mkCADerivation {
-    name = "dep";
-    inherit seed;
-    buildCommand = ''
-      echo ${currentTime} > $out
-    '';
-  };
-  dep1 = dep 1;
-  dep2 = dep 2;
-  toplevel = mkCADerivation {
-    name = "toplevel";
-    buildCommand = ''
-      test ${dep1} == ${dep2}
-      touch $out
-    '';
-  };
-}
-

tests/ca/post-hook.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-source common.sh
-
-sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
-
-export NIX_TESTS_CA_BY_DEFAULT=1
-cd ..
-source ./post-hook.sh
-
-

tests/ca/recursive.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-source common.sh
-
-sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
-
-export NIX_TESTS_CA_BY_DEFAULT=1
-cd ..
-source ./recursive.sh
-
-

tests/ca/substitute.sh

@@ -17,15 +17,11 @@ buildDrvs () {
 
 # Populate the remote cache
 clearStore
-nix copy --to $REMOTE_STORE --file ./content-addressed.nix
+buildDrvs --post-build-hook ../push-to-store.sh
 
 # Restart the build on an empty store, ensuring that we don't build
 clearStore
-buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA
-# Check that the thing we’ve just substituted has its realisation stored
-nix realisation info --file ./content-addressed.nix transitivelyDependentCA
-# Check that its dependencies have it too
-nix realisation info --file ./content-addressed.nix dependentCA rootCA
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
 
 # Same thing, but
 # 1. With non-ca derivations

tests/config.nix.in

@@ -1,12 +1,3 @@
-let
-  contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1";
-  caArgs = if contentAddressedByDefault then {
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
-  } else {};
-in
-
 rec {
   shell = "@bash@";
 
@@ -22,6 +13,6 @@ rec {
       builder = shell;
       args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
-    } // caArgs // removeAttrs args ["builder" "meta"])
+    } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 }

tests/gc.sh

@@ -12,7 +12,7 @@ ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo
 nix-store --gc --print-roots | grep $outPath
 nix-store --gc --print-live | grep $outPath
 nix-store --gc --print-dead | grep $drvPath
-if nix-store --gc --print-dead | grep -E $outPath$; then false; fi
+if nix-store --gc --print-dead | grep $outPath; then false; fi
 
 nix-store --gc --print-dead
 

tests/local.mk

@@ -2,7 +2,6 @@ nix_tests = \
   hash.sh lang.sh add.sh simple.sh dependencies.sh \
   config.sh \
   gc.sh \
-  ca/gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
@@ -39,7 +38,6 @@ nix_tests = \
   search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
-  ca/post-hook.sh \
   function-trace.sh \
   recursive.sh \
   describe-stores.sh \
@@ -47,13 +45,9 @@ nix_tests = \
   build.sh \
   compute-levels.sh \
   ca/build.sh \
-  ca/build-with-garbage-path.sh \
-  ca/duplicate-realisation-in-closure.sh \
   ca/substitute.sh \
   ca/signatures.sh \
-  ca/nix-shell.sh \
   ca/nix-run.sh \
-  ca/recursive.sh \
   ca/nix-copy.sh
   # parallel.sh
 

tests/nix-shell.sh

@@ -2,20 +2,6 @@ source common.sh
 
 clearStore
 
-if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
-    nix-shell () {
-        command nix-shell --arg contentAddressed true "$@"
-    }
-
-    nix_develop() {
-        nix develop --arg contentAddressed true "$@"
-    }
-else
-    nix_develop() {
-        nix develop "$@"
-    }
-fi
-
 # Test nix-shell -A
 export IMPURE_VAR=foo
 export SELECTED_IMPURE_VAR=baz
@@ -55,7 +41,7 @@ output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(f
 [ "$output" = "foo bar" ]
 
 # Test nix-shell shebang mode
-sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
 chmod a+rx $TEST_ROOT/shell.shebang.sh
 
 output=$($TEST_ROOT/shell.shebang.sh abc def)
@@ -63,7 +49,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def)
 
 # Test nix-shell shebang mode again with metacharacters in the filename.
 # First word of filename is chosen to not match any file in the test root.
-sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
@@ -72,7 +58,7 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
 # Test nix-shell shebang mode for ruby
 # This uses a fake interpreter that returns the arguments passed
 # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
-sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
 chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
@@ -80,20 +66,20 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test nix-shell shebang mode for ruby again with metacharacters in the filename.
 # Note: fake interpreter only space-separates args without adding escapes to its output.
-sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
 [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
 
 # Test 'nix develop'.
-nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
 
 # Ensure `nix develop -c` preserves stdin
 echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 
 # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
-nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
+nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 
 # Test 'nix print-dev-env'.
 source <(nix print-dev-env -f shell.nix shellDrv)

tests/post-hook.sh

@@ -4,7 +4,7 @@ clearStore
 
 rm -f $TEST_ROOT/result
 
-export REMOTE_STORE=file:$TEST_ROOT/remote_store
+export REMOTE_STORE=$TEST_ROOT/remote_store
 
 # Build the dependencies and push them to the remote store
 nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh

tests/recursive.sh

@@ -9,9 +9,9 @@ rm -f $TEST_ROOT/result
 
 export unreachable=$(nix store add-path ./recursive.sh)
 
-NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
+NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
   with import ./config.nix;
-  mkDerivation rec {
+  mkDerivation {
     name = "recursive";
     dummy = builtins.toFile "dummy" "bla bla";
     SHELL = shell;
@@ -19,13 +19,11 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-com
     # Note: this is a string without context.
     unreachable = builtins.getEnv "unreachable";
 
-    NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT";
-
     requiredSystemFeatures = [ "recursive-nix" ];
 
     buildCommand = '\'\''
       mkdir $out
-      opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}"
+      opts="--experimental-features nix-command"
 
       PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
 
@@ -48,15 +46,16 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-com
       # Add it to our closure.
       ln -s $foobar $out/foobar
 
-      [[ $(nix $opts path-info --all | wc -l) -eq 4 ]]
+      [[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
 
       # Build a derivation.
       nix $opts build -L --impure --expr '\''
-        with import ${./config.nix};
-        mkDerivation {
+        derivation {
           name = "inner1";
-          buildCommand = "echo $fnord blaat > $out";
+          builder = builtins.getEnv "SHELL";
+          system = builtins.getEnv "system";
           fnord = builtins.toFile "fnord" "fnord";
+          args = [ "-c" "echo $fnord blaat > $out" ];
         }
       '\''
 

tests/shell.nix

@@ -1,18 +1,6 @@
-{ inNixShell ? false, contentAddressed ? false }:
+{ inNixShell ? false }:
 
-let cfg = import ./config.nix; in
-with cfg;
-
-let
-  mkDerivation =
-    if contentAddressed then
-      args: cfg.mkDerivation ({
-        __contentAddressed = true;
-        outputHashMode = "recursive";
-        outputHashAlgo = "sha256";
-      } // args)
-    else cfg.mkDerivation;
-in
+with import ./config.nix;
 
 let pkgs = rec {
   setupSh = builtins.toFile "setup" ''

thingToBench.nix

@@ -0,0 +1,12 @@
+let
+  # pkgs = (builtins.getFlake "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31").legacyPackages.x86_64-linux;
+  pkgs = import <nixpkgs> {};
+in
+  pkgs.runCommandNoCC "foo" {
+    buildInputs = [
+      pkgs.firefox
+      pkgs.pandoc
+      # pkgs.nixosTests.xfce
+    ];
+  }
+  "echo bar > $out"