Use Boost's regex implementation instead of libstdc++'s

The latter runs in unbounded stack space so it crashes on large
inputs.

Fixes #4550.

doc/manual/generate-manpage.nix

@@ -7,7 +7,10 @@ let
 
   showCommand =
     { command, def, filename }:
-    "# Name\n\n"
+    ''
+      **Warning**: This program is **experimental** and its interface is subject to change.
+    ''
+    + "# Name\n\n"
     + "`${command}` - ${def.description}\n\n"
     + "# Synopsis\n\n"
     + showSynopsis { inherit command; args = def.args; }

doc/manual/src/command-ref/nix-shell.md

@@ -232,22 +232,23 @@ terraform apply
 > in a nix-shell shebang.
 
 Finally, using the merging of multiple nix-shell shebangs the following
-Haskell script uses a specific branch of Nixpkgs/NixOS (the 18.03 stable
+Haskell script uses a specific branch of Nixpkgs/NixOS (the 20.03 stable
 branch):
 
 ```haskell
 #! /usr/bin/env nix-shell
-#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
-#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
+#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
+#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
 
-import Network.HTTP
+import Network.Curl.Download
 import Text.HTML.TagSoup
+import Data.Either
+import Data.ByteString.Char8 (unpack)
 
 -- Fetch nixos.org and print all hrefs.
 main = do
-  resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/")
-  body <- getResponseBody resp
-  let tags = filter (isTagOpenName "a") $ parseTags body
+  resp <- openURI "https://nixos.org/"
+  let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp
   let tags' = map (fromAttrib "href") tags
   mapM_ putStrLn $ filter (/= "") tags'
 ```

flake.nix

@@ -135,7 +135,7 @@
               # Copy libboost_context so we don't get all of Boost in our closure.
               # https://github.com/NixOS/nixpkgs/issues/45462
               mkdir -p $out/lib
-              cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+              cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib
               rm -f $out/lib/*.a
               ${lib.optionalString stdenv.isLinux ''
                 chmod u+w $out/lib/*.so.*

src/libcmd/command.hh

@@ -48,8 +48,6 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs
     ref<EvalState> getEvalState();
 
     std::shared_ptr<EvalState> evalState;
-
-    ~EvalCommand();
 };
 
 struct MixFlakeOptions : virtual Args, EvalCommand

src/libcmd/installables.cc

@@ -9,6 +9,7 @@
 #include "store-api.hh"
 #include "shared.hh"
 #include "flake/flake.hh"
+#include "eval-cache.hh"
 #include "url.hh"
 #include "registry.hh"
 
@@ -221,8 +222,10 @@ void completeFlakeRefWithFragment(
             // FIXME: do tilde expansion.
             auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
 
-            auto lockedFlake = lockFlake(*evalState, flakeRef, lockFlags);
-            auto rootValue = getFlakeValue(*evalState, lockedFlake);
+            auto evalCache = openEvalCache(*evalState,
+                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
+
+            auto root = evalCache->getRoot();
 
             /* Complete 'fragment' relative to all the
                attrpath prefixes as well as the root of the
@@ -240,18 +243,12 @@ void completeFlakeRefWithFragment(
                     attrPath.pop_back();
                 }
 
-                auto cachedFieldNames = rootValue->getCache().listChildrenAtPath(evalState->symbols, attrPath);
+                auto attr = root->findAlongAttrPath(attrPath);
+                if (!attr) continue;
 
-                if (!cachedFieldNames) {
-                    auto accessResult = evalState->getOptionalAttrField(*rootValue, attrPath, noPos);
-                    if (accessResult.error) continue;
-                    cachedFieldNames = evalState->listAttrFields(*accessResult.value, *accessResult.pos);
-                }
-
-                for (auto & lastFieldName : *cachedFieldNames) {
-                    if (hasPrefix(lastFieldName, lastAttr)) {
-                        auto attrPath2 = attrPath;
-                        attrPath2.push_back(lastFieldName);
+                for (auto & attr2 : attr->getAttrs()) {
+                    if (hasPrefix(attr2, lastAttr)) {
+                        auto attrPath2 = attr->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
                         completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
@@ -263,8 +260,8 @@ void completeFlakeRefWithFragment(
                attrpaths. */
             if (fragment.empty()) {
                 for (auto & attrPath : defaultFlakeAttrPaths) {
-                    auto accessResult = evalState->getOptionalAttrField(*rootValue, {evalState->symbols.create(attrPath)}, noPos);
-                    if (accessResult.error) continue;
+                    auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
+                    if (!attr) continue;
                     completions->add(flakeRefS + "#");
                 }
             }
@@ -283,12 +280,6 @@ ref<EvalState> EvalCommand::getEvalState()
     return ref<EvalState>(evalState);
 }
 
-EvalCommand::~EvalCommand()
-{
-    if (evalState)
-        evalState->printStats();
-}
-
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")
@@ -320,6 +311,24 @@ Buildable Installable::toBuildable()
     return std::move(buildables[0]);
 }
 
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+Installable::getCursors(EvalState & state)
+{
+    auto evalCache =
+        std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
+            [&]() { return toValue(state).first; });
+    return {{evalCache->getRoot(), ""}};
+}
+
+std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+Installable::getCursor(EvalState & state)
+{
+    auto cursors = getCursors(state);
+    if (cursors.empty())
+        throw Error("cannot find flake attribute '%s'", what());
+    return cursors[0];
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -390,11 +399,11 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    std::vector<Installable::ValueInfo> toValues(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
         state.forceValue(*vRes);
-        return {{vRes, pos, attrPath}};
+        return {vRes, pos};
     }
 
     virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -402,7 +411,7 @@ struct InstallableAttrPath : InstallableValue
 
 std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations()
 {
-    auto v = toValue(*state).value;
+    auto v = toValue(*state).first;
 
     Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
@@ -436,7 +445,45 @@ std::vector<std::string> InstallableFlake::getActualAttrPaths()
 
 Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
 {
-    return getFlakeValue(state, lockedFlake);
+    auto vFlake = state.allocValue();
+
+    callFlake(state, lockedFlake, *vFlake);
+
+    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    assert(aOutputs);
+
+    state.forceValue(*aOutputs->value);
+
+    return aOutputs->value;
+}
+
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    std::shared_ptr<flake::LockedFlake> lockedFlake)
+{
+    auto fingerprint = lockedFlake->getFingerprint();
+    return make_ref<nix::eval_cache::EvalCache>(
+        evalSettings.useEvalCache && evalSettings.pureEval
+            ? std::optional { std::cref(fingerprint) }
+            : std::nullopt,
+        state,
+        [&state, lockedFlake]()
+        {
+            /* For testing whether the evaluation cache is
+               complete. */
+            if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
+                throw Error("not everything is cached, but evaluation is not allowed");
+
+            auto vFlake = state.allocValue();
+            flake::callFlake(state, *lockedFlake, *vFlake);
+
+            state.forceAttrs(*vFlake);
+
+            auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+            assert(aOutputs);
+
+            return aOutputs->value;
+        });
 }
 
 static std::string showAttrPaths(const std::vector<std::string> & paths)
@@ -453,20 +500,29 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
 {
     auto lockedFlake = getLockedFlake();
 
-    auto flakeValue = toValue(*state);
+    auto cache = openEvalCache(*state, lockedFlake);
+    auto root = cache->getRoot();
 
-    auto rawDrvInfo = getDerivation(*state, *flakeValue.value, false);
-    if (!rawDrvInfo)
-        throw Error("flake output attribute '%s' is not a derivation", flakeValue.positionInfo);
+    for (auto & attrPath : getActualAttrPaths()) {
+        auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
+        if (!attr) continue;
 
-    auto drvInfo = InstallableValue::DerivationInfo
-    {
-        state->store->parseStorePath(rawDrvInfo->queryDrvPath()),
-            state->store->maybeParseStorePath(rawDrvInfo->queryOutPath()),
-            rawDrvInfo->queryOutputName()
-    };
+        if (!attr->isDerivation())
+            throw Error("flake output attribute '%s' is not a derivation", attrPath);
 
-    return {flakeValue.positionInfo, lockedFlake->flake.lockedRef, std::move(drvInfo)};
+        auto drvPath = attr->forceDerivation();
+
+        auto drvInfo = DerivationInfo{
+            std::move(drvPath),
+            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
+            attr->getAttr(state->sOutputName)->getString()
+        };
+
+        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
+    }
+
+    throw Error("flake '%s' does not provide attribute %s",
+        flakeRef, showAttrPaths(getActualAttrPaths()));
 }
 
 std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
@@ -476,20 +532,8 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
     return res;
 }
 
-Installable::ValueInfo InstallableFlake::toValue(EvalState & state)
+std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 {
-    auto values = toValues(state);
-    if (values.empty())
-        throw Error("flake '%s' does not provide attribute %s",
-            flakeRef, showAttrPaths(getActualAttrPaths()));
-    return values[0];
-
-}
-
-std::vector<Installable::ValueInfo>
-InstallableFlake::toValues(EvalState & state)
-{
-    std::vector<Installable::ValueInfo> res;
     auto lockedFlake = getLockedFlake();
 
     auto vOutputs = getFlakeOutputs(state, *lockedFlake);
@@ -500,11 +544,30 @@ InstallableFlake::toValues(EvalState & state)
         try {
             auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
             state.forceValue(*v);
-            res.push_back({v, pos, attrPath});
+            return {v, pos};
         } catch (AttrPathNotFound & e) {
         }
     }
 
+    throw Error("flake '%s' does not provide attribute %s",
+        flakeRef, showAttrPaths(getActualAttrPaths()));
+}
+
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+InstallableFlake::getCursors(EvalState & state)
+{
+    auto evalCache = openEvalCache(state,
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
+
+    auto root = evalCache->getRoot();
+
+    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
+
+    for (auto & attrPath : getActualAttrPaths()) {
+        auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
+        if (attr) res.push_back({attr, attrPath});
+    }
+
     return res;
 }
 

src/libcmd/installables.hh

@@ -54,23 +54,7 @@ struct Installable
 
     App toApp(EvalState & state);
 
-    struct ValueInfo {
-        Value * value;
-        Pos pos;
-        string positionInfo;
-    };
-
-    virtual ValueInfo toValue(EvalState & state)
-    {
-        auto values = toValues(state);
-        if (values.empty())
-            throw Error("Installable '%s' does not provide a default value",
-                    what());
-        return values[0];
-    }
-
-    virtual std::vector<ValueInfo>
-    toValues(EvalState & state)
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }
@@ -82,6 +66,11 @@ struct Installable
         return {};
     }
 
+    virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    getCursors(EvalState & state);
+
+    std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+    getCursor(EvalState & state);
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
@@ -131,14 +120,18 @@ struct InstallableFlake : InstallableValue
 
     std::vector<DerivationInfo> toDerivations() override;
 
-    ValueInfo toValue(EvalState & state) override;
+    std::pair<Value *, Pos> toValue(EvalState & state) override;
 
-    std::vector<ValueInfo>
-    toValues(EvalState & state) override;
+    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    getCursors(EvalState & state) override;
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
     FlakeRef nixpkgsFlakeRef() const override;
 };
 
+ref<eval_cache::EvalCache> openEvalCache(
+    EvalState & state,
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
+
 }

src/libexpr/attr-path.cc

@@ -73,12 +73,11 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            Value * vRes = state.allocValue();
-            auto getResult = state.getOptionalAttrField(*v, {state.symbols.create(attr)}, pos, *vRes);
-            if (getResult.error)
+            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
+            if (a == v->attrs->end())
                 throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
-            v = vRes;
-            pos = *getResult.pos;
+            v = &*a->value;
+            pos = *a->pos;
         }
 
         else {

src/libexpr/attr-set.hh

@@ -38,10 +38,6 @@ public:
 
 private:
     size_t size_, capacity_;
-
-public:
-
-private:
     Attr attrs[0];
 
     Bindings(size_t capacity) : size_(0), capacity_(capacity) { }

src/libexpr/context.cc

@@ -1,21 +0,0 @@
-#include "context.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
-    } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
-}
-
-std::string encodeContext(std::string_view name, std::string_view path)
-{
-    return "!" + std::string(name) + "!" + std::string(path);
-}
-
-}

src/libexpr/context.hh

@@ -1,11 +0,0 @@
-#include "util.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s);
-
-std::string encodeContext(std::string_view name, std::string_view path);
-
-}

src/libexpr/eval-cache.cc

@@ -0,0 +1,629 @@
+#include "eval-cache.hh"
+#include "sqlite.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+
+namespace nix::eval_cache {
+
+static const char * schema = R"sql(
+create table if not exists Attributes (
+    parent      integer not null,
+    name        text,
+    type        integer not null,
+    value       text,
+    context     text,
+    primary key (parent, name)
+);
+)sql";
+
+struct AttrDb
+{
+    std::atomic_bool failed{false};
+
+    struct State
+    {
+        SQLite db;
+        SQLiteStmt insertAttribute;
+        SQLiteStmt insertAttributeWithContext;
+        SQLiteStmt queryAttribute;
+        SQLiteStmt queryAttributes;
+        std::unique_ptr<SQLiteTxn> txn;
+    };
+
+    std::unique_ptr<Sync<State>> _state;
+
+    AttrDb(const Hash & fingerprint)
+        : _state(std::make_unique<Sync<State>>())
+    {
+        auto state(_state->lock());
+
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+        createDirs(cacheDir);
+
+        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+
+        state->db = SQLite(dbPath);
+        state->db.isCache();
+        state->db.exec(schema);
+
+        state->insertAttribute.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
+
+        state->insertAttributeWithContext.create(state->db,
+            "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
+
+        state->queryAttribute.create(state->db,
+            "select rowid, type, value, context from Attributes where parent = ? and name = ?");
+
+        state->queryAttributes.create(state->db,
+            "select name from Attributes where parent = ?");
+
+        state->txn = std::make_unique<SQLiteTxn>(state->db);
+    }
+
+    ~AttrDb()
+    {
+        try {
+            auto state(_state->lock());
+            if (!failed)
+                state->txn->commit();
+            state->txn.reset();
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    template<typename F>
+    AttrId doSQLite(F && fun)
+    {
+        if (failed) return 0;
+        try {
+            return fun();
+        } catch (SQLiteError &) {
+            ignoreException();
+            failed = true;
+            return 0;
+        }
+    }
+
+    AttrId setAttrs(
+        AttrKey key,
+        const std::vector<Symbol> & attrs)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::FullAttrs)
+                (0, false).exec();
+
+            AttrId rowId = state->db.getLastInsertedRowId();
+            assert(rowId);
+
+            for (auto & attr : attrs)
+                state->insertAttribute.use()
+                    (rowId)
+                    (attr)
+                    (AttrType::Placeholder)
+                    (0, false).exec();
+
+            return rowId;
+        });
+    }
+
+    AttrId setString(
+        AttrKey key,
+        std::string_view s,
+        const char * * context = nullptr)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            if (context) {
+                std::string ctx;
+                for (const char * * p = context; *p; ++p) {
+                    if (p != context) ctx.push_back(' ');
+                    ctx.append(*p);
+                }
+                state->insertAttributeWithContext.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                    (s)
+                    (ctx).exec();
+            } else {
+                state->insertAttribute.use()
+                    (key.first)
+                    (key.second)
+                    (AttrType::String)
+                (s).exec();
+            }
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setBool(
+        AttrKey key,
+        bool b)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Bool)
+                (b ? 1 : 0).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setPlaceholder(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Placeholder)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setMissing(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Missing)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setMisc(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Misc)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    AttrId setFailed(AttrKey key)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (key.second)
+                (AttrType::Failed)
+                (0, false).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> getAttr(
+        AttrKey key,
+        SymbolTable & symbols)
+    {
+        auto state(_state->lock());
+
+        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+        if (!queryAttribute.next()) return {};
+
+        auto rowId = (AttrType) queryAttribute.getInt(0);
+        auto type = (AttrType) queryAttribute.getInt(1);
+
+        switch (type) {
+            case AttrType::Placeholder:
+                return {{rowId, placeholder_t()}};
+            case AttrType::FullAttrs: {
+                // FIXME: expensive, should separate this out.
+                std::vector<Symbol> attrs;
+                auto queryAttributes(state->queryAttributes.use()(rowId));
+                while (queryAttributes.next())
+                    attrs.push_back(symbols.create(queryAttributes.getStr(0)));
+                return {{rowId, attrs}};
+            }
+            case AttrType::String: {
+                std::vector<std::pair<Path, std::string>> context;
+                if (!queryAttribute.isNull(3))
+                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
+                        context.push_back(decodeContext(s));
+                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
+            }
+            case AttrType::Bool:
+                return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::Missing:
+                return {{rowId, missing_t()}};
+            case AttrType::Misc:
+                return {{rowId, misc_t()}};
+            case AttrType::Failed:
+                return {{rowId, failed_t()}};
+            default:
+                throw Error("unexpected type in evaluation cache");
+        }
+    }
+};
+
+static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+{
+    try {
+        return std::make_shared<AttrDb>(fingerprint);
+    } catch (SQLiteError &) {
+        ignoreException();
+        return nullptr;
+    }
+}
+
+EvalCache::EvalCache(
+    std::optional<std::reference_wrapper<const Hash>> useCache,
+    EvalState & state,
+    RootLoader rootLoader)
+    : db(useCache ? makeAttrDb(*useCache) : nullptr)
+    , state(state)
+    , rootLoader(rootLoader)
+{
+}
+
+Value * EvalCache::getRootValue()
+{
+    if (!value) {
+        debug("getting root value");
+        value = allocRootValue(rootLoader());
+    }
+    return *value;
+}
+
+std::shared_ptr<AttrCursor> EvalCache::getRoot()
+{
+    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+}
+
+AttrCursor::AttrCursor(
+    ref<EvalCache> root,
+    Parent parent,
+    Value * value,
+    std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
+    : root(root), parent(parent), cachedValue(std::move(cachedValue))
+{
+    if (value)
+        _value = allocRootValue(value);
+}
+
+AttrKey AttrCursor::getKey()
+{
+    if (!parent)
+        return {0, root->state.sEpsilon};
+    if (!parent->first->cachedValue) {
+        parent->first->cachedValue = root->db->getAttr(
+            parent->first->getKey(), root->state.symbols);
+        assert(parent->first->cachedValue);
+    }
+    return {parent->first->cachedValue->first, parent->second};
+}
+
+Value & AttrCursor::getValue()
+{
+    if (!_value) {
+        if (parent) {
+            auto & vParent = parent->first->getValue();
+            root->state.forceAttrs(vParent);
+            auto attr = vParent.attrs->get(parent->second);
+            if (!attr)
+                throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
+            _value = allocRootValue(attr->value);
+        } else
+            _value = allocRootValue(root->getRootValue());
+    }
+    return **_value;
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath() const
+{
+    if (parent) {
+        auto attrPath = parent->first->getAttrPath();
+        attrPath.push_back(parent->second);
+        return attrPath;
+    } else
+        return {};
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
+{
+    auto attrPath = getAttrPath();
+    attrPath.push_back(name);
+    return attrPath;
+}
+
+std::string AttrCursor::getAttrPathStr() const
+{
+    return concatStringsSep(".", getAttrPath());
+}
+
+std::string AttrCursor::getAttrPathStr(Symbol name) const
+{
+    return concatStringsSep(".", getAttrPath(name));
+}
+
+Value & AttrCursor::forceValue()
+{
+    debug("evaluating uncached attribute %s", getAttrPathStr());
+
+    auto & v = getValue();
+
+    try {
+        root->state.forceValue(v);
+    } catch (EvalError &) {
+        debug("setting '%s' to failed", getAttrPathStr());
+        if (root->db)
+            cachedValue = {root->db->setFailed(getKey()), failed_t()};
+        throw;
+    }
+
+    if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
+        if (v.type() == nString)
+            cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
+                           string_t{v.string.s, {}}};
+        else if (v.type() == nPath)
+            cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
+        else if (v.type() == nBool)
+            cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+        else if (v.type() == nAttrs)
+            ; // FIXME: do something?
+        else
+            cachedValue = {root->db->setMisc(getKey()), misc_t()};
+    }
+
+    return v;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+
+        if (cachedValue) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                for (auto & attr : *attrs)
+                    if (attr == name)
+                        return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
+                return nullptr;
+            } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
+                auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
+                if (attr) {
+                    if (std::get_if<missing_t>(&attr->second))
+                        return nullptr;
+                    else if (std::get_if<failed_t>(&attr->second)) {
+                        if (forceErrors)
+                            debug("reevaluating failed cached attribute '%s'");
+                        else
+                            throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+                    } else
+                        return std::make_shared<AttrCursor>(root,
+                            std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
+                }
+                // Incomplete attrset, so need to fall thru and
+                // evaluate to see whether 'name' exists
+            } else
+                return nullptr;
+                //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nAttrs)
+        return nullptr;
+        //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    auto attr = v.attrs->get(name);
+
+    if (!attr) {
+        if (root->db) {
+            if (!cachedValue)
+                cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+            root->db->setMissing({cachedValue->first, name});
+        }
+        return nullptr;
+    }
+
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+        cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
+    }
+
+    return std::make_shared<AttrCursor>(
+        root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
+{
+    return maybeGetAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+{
+    auto p = maybeGetAttr(name, forceErrors);
+    if (!p)
+        throw Error("attribute '%s' does not exist", getAttrPathStr(name));
+    return p;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+{
+    return getAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+{
+    auto res = shared_from_this();
+    for (auto & attr : attrPath) {
+        res = res->maybeGetAttr(attr);
+        if (!res) return {};
+    }
+    return res;
+}
+
+std::string AttrCursor::getString()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                debug("using cached string attribute '%s'", getAttrPathStr());
+                return s->first;
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nString && v.type() != nPath)
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
+
+    return v.type() == nString ? v.string.s : v.path;
+}
+
+string_t AttrCursor::getStringWithContext()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+                bool valid = true;
+                for (auto & c : s->second) {
+                    if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+                        valid = false;
+                        break;
+                    }
+                }
+                if (valid) {
+                    debug("using cached string attribute '%s'", getAttrPathStr());
+                    return *s;
+                }
+            } else
+                throw TypeError("'%s' is not a string", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() == nString)
+        return {v.string.s, v.getContext()};
+    else if (v.type() == nPath)
+        return {v.path, {}};
+    else
+        throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
+}
+
+bool AttrCursor::getBool()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto b = std::get_if<bool>(&cachedValue->second)) {
+                debug("using cached Boolean attribute '%s'", getAttrPathStr());
+                return *b;
+            } else
+                throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nBool)
+        throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+
+    return v.boolean;
+}
+
+std::vector<Symbol> AttrCursor::getAttrs()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+                debug("using cached attrset attribute '%s'", getAttrPathStr());
+                return *attrs;
+            } else
+                throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nAttrs)
+        throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+    std::vector<Symbol> attrs;
+    for (auto & attr : *getValue().attrs)
+        attrs.push_back(attr.name);
+    std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
+        return (const string &) a < (const string &) b;
+    });
+
+    if (root->db)
+        cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
+
+    return attrs;
+}
+
+bool AttrCursor::isDerivation()
+{
+    auto aType = maybeGetAttr("type");
+    return aType && aType->getString() == "derivation";
+}
+
+StorePath AttrCursor::forceDerivation()
+{
+    auto aDrvPath = getAttr(root->state.sDrvPath, true);
+    auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+    if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
+        /* The eval cache contains 'drvPath', but the actual path has
+           been garbage-collected. So force it to be regenerated. */
+        aDrvPath->forceValue();
+        if (!root->state.store->isValidPath(drvPath))
+            throw Error("don't know how to recreate store derivation '%s'!",
+                root->state.store->printStorePath(drvPath));
+    }
+    return drvPath;
+}
+
+}

src/libexpr/eval-cache.hh

@@ -0,0 +1,123 @@
+#pragma once
+
+#include "sync.hh"
+#include "hash.hh"
+#include "eval.hh"
+
+#include <functional>
+#include <variant>
+
+namespace nix::eval_cache {
+
+MakeError(CachedEvalError, EvalError);
+
+struct AttrDb;
+class AttrCursor;
+
+class EvalCache : public std::enable_shared_from_this<EvalCache>
+{
+    friend class AttrCursor;
+
+    std::shared_ptr<AttrDb> db;
+    EvalState & state;
+    typedef std::function<Value *()> RootLoader;
+    RootLoader rootLoader;
+    RootValue value;
+
+    Value * getRootValue();
+
+public:
+
+    EvalCache(
+        std::optional<std::reference_wrapper<const Hash>> useCache,
+        EvalState & state,
+        RootLoader rootLoader);
+
+    std::shared_ptr<AttrCursor> getRoot();
+};
+
+enum AttrType {
+    Placeholder = 0,
+    FullAttrs = 1,
+    String = 2,
+    Missing = 3,
+    Misc = 4,
+    Failed = 5,
+    Bool = 6,
+};
+
+struct placeholder_t {};
+struct missing_t {};
+struct misc_t {};
+struct failed_t {};
+typedef uint64_t AttrId;
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+
+typedef std::variant<
+    std::vector<Symbol>,
+    string_t,
+    placeholder_t,
+    missing_t,
+    misc_t,
+    failed_t,
+    bool
+    > AttrValue;
+
+class AttrCursor : public std::enable_shared_from_this<AttrCursor>
+{
+    friend class EvalCache;
+
+    ref<EvalCache> root;
+    typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
+    Parent parent;
+    RootValue _value;
+    std::optional<std::pair<AttrId, AttrValue>> cachedValue;
+
+    AttrKey getKey();
+
+    Value & getValue();
+
+public:
+
+    AttrCursor(
+        ref<EvalCache> root,
+        Parent parent,
+        Value * value = nullptr,
+        std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
+
+    std::vector<Symbol> getAttrPath() const;
+
+    std::vector<Symbol> getAttrPath(Symbol name) const;
+
+    std::string getAttrPathStr() const;
+
+    std::string getAttrPathStr(Symbol name) const;
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
+
+    std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+
+    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+
+    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
+
+    std::string getString();
+
+    string_t getStringWithContext();
+
+    bool getBool();
+
+    std::vector<Symbol> getAttrs();
+
+    bool isDerivation();
+
+    Value & forceValue();
+
+    /* Force creation of the .drv file in the Nix store. */
+    StorePath forceDerivation();
+};
+
+}

src/libexpr/eval.cc

@@ -433,9 +433,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    for (auto [_, cache] : evalCache) {
-        cache->commit();
-    }
 }
 
 
@@ -1121,396 +1118,48 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa
 
 unsigned long nrLookups = 0;
 
-tree_cache::AttrValue cachedValueFor(const Value& v)
-{
-    tree_cache::AttrValue valueToCache;
-    switch (v.type()) {
-        case nThunk:
-            valueToCache = tree_cache::thunk_t{};
-            break;
-        case nNull:
-        case nList:
-        case nFunction:
-        case nExternal:
-            valueToCache = tree_cache::unknown_t{};
-            break;
-        case nBool:
-            valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
-            break;
-        case nString:
-            valueToCache = tree_cache::string_t{
-                v.string.s,
-                v.getContext()
-            };
-            break;
-        case nPath:
-            valueToCache = tree_cache::string_t{
-                v.path,
-                {}
-            };
-            break;
-        case nAttrs:
-            valueToCache = tree_cache::attributeSet_t{};
-            break;
-        case nInt:
-            valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
-            break;
-        case nFloat:
-            valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
-            break;
-    };
-    return valueToCache;
-}
-
-std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
-{
-    auto ret = std::vector<Symbol>();
-    if (rawCache) {
-        auto cachedValue = rawCache->getCachedValue();
-            if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
-                for (auto & fieldStr : rawCache->getChildren())
-                    ret.push_back(symbols.create(fieldStr));
-            }
-            return ret;
-    }
-    return std::nullopt;
-}
-
-std::optional<std::vector<Symbol>> ValueCache::listChildrenAtPath(SymbolTable & symbols, const std::vector<Symbol> & attrPath)
-{
-    auto ret = std::vector<Symbol>();
-    if (rawCache) {
-        auto rawChildren = rawCache->getChildrenAtPath(attrPath);
-        if (!rawChildren) return std::nullopt;
-        for (auto & fieldStr : rawChildren.value())
-            ret.push_back(symbols.create(fieldStr));
-        return ret;
-    }
-    return std::nullopt;
-}
-
-std::vector<Symbol> EvalState::listAttrFields(Value & attrs, const Pos & pos)
-{
-    // First try to get it from the cache
-    if (auto cachedRes = attrs.getCache().listChildren(symbols))
-        return cachedRes.value();
-
-    auto ret = std::vector<Symbol>();
-    forceAttrs(attrs, pos);
-    ret.reserve(attrs.attrs->size());
-    for (auto & attr : *attrs.attrs) {
-        ret.push_back(attr.name);
-    }
-    return ret;
-}
-
-EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
-{
-    Value* resValue = allocValue();
-    auto accessResult = getOptionalAttrField(attrs, selector, pos, *resValue);
-    accessResult.value = resValue;
-    return accessResult;
-}
-
-ValueCache::CacheResult ValueCache::getValue(Store & store, const std::vector<Symbol> & selector, Value & dest)
-{
-    if (!rawCache)
-        return { NoCacheKey };
-    auto resultingCursor = rawCache->findAlongAttrPath(selector);
-    if (!resultingCursor)
-        return { CacheMiss };
-
-    dest.setCache(ValueCache(resultingCursor));
-
-    auto cachedValue = resultingCursor->getCachedValue();
-    return std::visit(
-        overloaded{
-            [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
-            [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
-            [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
-            [&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
-            [&](tree_cache::missing_t x) {
-                return ValueCache::CacheResult{
-                    .returnCode = CacheHit,
-                    .lastQueriedSymbolIfMissing = x.attrName
-                };
-            },
-            [&](tree_cache::string_t s) {
-                PathSet context;
-                for (auto& [pathName, outputName] : s.second) {
-                    // If the cached value depends on some non-existent
-                    // path, we need to discard it and force the evaluation
-                    // to bring back the context in the store
-                    if (!store.isValidPath(
-                            store.parseStorePath(pathName)))
-                        return ValueCache::CacheResult{UnCacheable};
-                    context.insert("!" + outputName + "!" + pathName);
-                }
-                mkString(dest, s.first, context);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<bool> b) {
-                dest.mkBool(b.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<int64_t> i) {
-                dest.mkInt(i.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<double> d) {
-                dest.mkFloat(d.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-        },
-        cachedValue);
-}
-
-void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
-{
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            nrCacheHits++;
-            break;
-        case ValueCache::CacheMiss:
-            nrCacheMisses++;
-            break;
-        case ValueCache::UnCacheable:
-            nrUncacheable++;
-            break;
-        case ValueCache::NoCacheKey:
-            nrUncached++;
-            break;
-        case ValueCache::Forward:
-            nrCacheHits++;
-            break;
-    };
-}
-
-struct ExprCastedVar : Expr
-{
-    Value * v;
-    ExprCastedVar(Value * v) : v(v) {};
-    void show(std::ostream & str) const override {
-        std::set<const Value*> active;
-        printValue(str, active, *v);
-    }
-
-    void bindVars(const StaticEnv & env) override {}
-    void eval(EvalState & state, Env & env, Value & v) override {
-        v = std::move(*this->v);
-    }
-    Value * maybeThunk(EvalState & state, Env & env) override {
-        return v;
-    }
-};
-
-EvalState::AttrAccesResult EvalState::lazyGetOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
-{
-    Value * dest = allocValue();
-    auto evalCache = attrs.getCache();
-    auto cacheResult = evalCache.getValue(*store, selector, *dest);
-    updateCacheStats(cacheResult);
-
-    if (cacheResult.returnCode == ValueCache::CacheHit) {
-        if (cacheResult.lastQueriedSymbolIfMissing)
-            return {
-                .error =
-                    AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing},
-                .pos = &pos,
-            };
-        else
-            return {.pos = &pos, .value = dest};
-    }
-
-    auto & newEnv(allocEnv(0));
-
-    auto recordAsVar = new ExprCastedVar(&attrs);
-
-    auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
-    dest->mkThunk (&newEnv, accessExpr);
-
-    return {
-        .pos = &pos,
-        .value = dest,
-    };
-
-}
-
-EvalState::AttrAccesResult EvalState::getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    auto evalCache = attrs.getCache();
-
-    if (auto maybeRawVal = evalCache.getRawValue();
-            maybeRawVal.has_value() &&
-            !std::holds_alternative<tree_cache::attributeSet_t>(maybeRawVal.value()) &&
-            !std::holds_alternative<tree_cache::thunk_t>(maybeRawVal.value())
-        )
-        return {
-            .error = AttrAccessError{
-                .attrName = selector[0],
-                .illTypedValue = &attrs,
-            },
-            .pos = &pos,
-        };
-
-    auto cacheResult = evalCache.getValue(*store, selector, dest);
-    updateCacheStats(cacheResult);
-
-    if (cacheResult.returnCode == ValueCache::CacheHit) {
-        if (cacheResult.lastQueriedSymbolIfMissing)
-            return {
-                .error =
-                    AttrAccessError{*cacheResult.lastQueriedSymbolIfMissing},
-                .pos = &pos,
-            };
-        else
-            return {.pos = &pos, .value = &dest};
-    }
-
-    const Pos * pos2 = &pos;
-    Value * currentValue = &attrs;
-    forceValue(*currentValue, *pos2);
-
-    auto resultingCursor = evalCache;
-    for (auto & name : selector) {
-        nrLookups++;
-        Bindings::iterator j;
-        if (currentValue->type() != nAttrs)
-            return {
-                .error = AttrAccessError{
-                    .attrName = name,
-                    .illTypedValue = currentValue,
-                },
-                .pos = pos2,
-            };
-        if ((j = currentValue->attrs->find(name)) == currentValue->attrs->end())
-            return {.error = AttrAccessError { name }, .pos = pos2 };
-        currentValue = j->value;
-        pos2 = j->pos;
-        try {
-            forceValue(*currentValue, *pos2);
-        } catch (EvalError & e) {
-            resultingCursor.addFailedChild(name, e);
-            throw;
-        };
-        if (cacheResult.returnCode == ValueCache::CacheMiss) {
-            resultingCursor = resultingCursor.addChild(name, *currentValue);
-            currentValue->setCache(resultingCursor);
-        }
-        if (countCalls && pos2) attrSelects[*pos2]++;
-    }
-
-    if (cacheResult.returnCode != ValueCache::CacheMiss) {
-        resultingCursor = dest.getCache();
-        currentValue->setCache(resultingCursor);
-    }
-
-    dest = *currentValue;
-    return { .pos = pos2 };
-}
-
-const ValueCache ValueCache::empty = ValueCache(nullptr);
-
-std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
-{
-    if (!rawCache)
-        return std::nullopt;
-    return rawCache->getCachedValue();
-}
-
-ValueCache ValueCache::addChild(const Symbol& name, const Value& value)
-{
-    if (!rawCache)
-        return ValueCache::empty;
-
-    auto cachedValue = cachedValueFor(value);
-    auto ret = ValueCache(rawCache->addChild(name, cachedValue));
-    if (value.type() == nAttrs)
-        ret.addAttrSetChilds(*value.attrs);
-    return ret;
-}
-
-ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
-{
-    if (!rawCache) return ValueCache::empty;
-    return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
-}
-
-ValueCache ValueCache::addNumChild(SymbolTable & symbols, int idx, const Value & value)
-{
-    return addChild(symbols.create(std::to_string(idx)), value);
-}
-
-void ValueCache::addAttrSetChilds(Bindings & children)
-{
-    if (!rawCache) return;
-    for (auto & attr : children) {
-        // We could in theory directly store the value, but that would cause
-        // an infinite recursion in case of a cyclic attrset.
-        // So in a first time, just store a thunk
-        rawCache->addChild(attr.name, tree_cache::thunk_t{});
-        /* addChild(attr.name, *attr.value); */
-    }
-
-}
-
-void ValueCache::addListChilds(SymbolTable & symbols, Value** elems, int listSize)
-{
-    if (!rawCache) return;
-    for (auto i = 0; i < listSize; i++) {
-        addNumChild(symbols, i, *elems[i]);
-    }
-}
-
-void EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    auto missingFieldInfo = getOptionalAttrField(attrs, selector, pos, dest);
-    if (missingFieldInfo.error) {
-        if (missingFieldInfo.error->illTypedValue) {
-            throwTypeError("value is %1% while a set was expected", **missingFieldInfo.error->illTypedValue);
-        } else {
-            throwEvalError(
-                    *missingFieldInfo.pos,
-                    "attribute '%1%' missing", missingFieldInfo.error->attrName
-            );
-        }
-    }
-}
-
-Value* EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos)
-{
-    Value* res = allocValue();
-    getAttrField(attrs, selector, pos, *res);
-    return res;
-}
-
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
+    Pos * pos2 = 0;
+    Value * vAttrs = &vTmp;
 
     e->eval(state, env, vTmp);
 
-    std::vector<Symbol> evaluatedAttrs;
-    for (auto & i : attrPath) {
-        evaluatedAttrs.push_back(getName(i, state, env));
-    }
-
     try {
-        if (def) {
-            auto missingFieldInfo = state.getOptionalAttrField(vTmp, evaluatedAttrs, pos, v);
-            if (missingFieldInfo.error) {
-                def->eval(state, env, v);
-                return;
+
+        for (auto & i : attrPath) {
+            nrLookups++;
+            Bindings::iterator j;
+            Symbol name = getName(i, state, env);
+            if (def) {
+                state.forceValue(*vAttrs, pos);
+                if (vAttrs->type() != nAttrs ||
+                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                {
+                    def->eval(state, env, v);
+                    return;
+                }
+            } else {
+                state.forceAttrs(*vAttrs, pos);
+                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                    throwEvalError(pos, "attribute '%1%' missing", name);
             }
-            return;
+            vAttrs = j->value;
+            pos2 = j->pos;
+            if (state.countCalls && pos2) state.attrSelects[*pos2]++;
         }
-        state.getAttrField(vTmp, evaluatedAttrs, pos, v);
+
+        state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
+
     } catch (Error & e) {
-        if (pos.file != state.sDerivationNix)
-            addErrorTrace(e, pos, "while evaluating the attribute '%1%'",
-                    showAttrPath(state, env, attrPath));
+        if (pos2 && pos2->file != state.sDerivationNix)
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
         throw;
     }
+
+    v = *vAttrs;
 }
 
 
@@ -2041,6 +1690,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -2049,7 +1710,7 @@ void copyContext(const Value & v, PathSet & context)
 }
 
 
-std::vector<std::pair<Path, std::string>> Value::getContext() const
+std::vector<std::pair<Path, std::string>> Value::getContext()
 {
     std::vector<std::pair<Path, std::string>> res;
     assert(internalType == tString);
@@ -2059,15 +1720,6 @@ std::vector<std::pair<Path, std::string>> Value::getContext() const
     return res;
 }
 
-void Value::setCache(ValueCache cache)
-{
-    evalCache = cache;
-}
-
-ValueCache Value::getCache() const
-{
-    return evalCache;
-}
 
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
@@ -2094,12 +1746,12 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
 
 bool EvalState::isDerivation(Value & v)
 {
-    auto typeAccesRes = getOptionalAttrField(v, {sType}, noPos);
-    if (typeAccesRes.error)
-        return false;
-    forceValue(*typeAccesRes.value);
-    if (typeAccesRes.value->type() != nString) return false;
-    return strcmp(typeAccesRes.value->string.s, "derivation") == 0;
+    if (v.type() != nAttrs) return false;
+    Bindings::iterator i = v.attrs->find(sType);
+    if (i == v.attrs->end()) return false;
+    forceValue(*i->value);
+    if (i->value->type() != nString) return false;
+    return strcmp(i->value->string.s, "derivation") == 0;
 }
 
 
@@ -2138,9 +1790,9 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
         if (maybeString) {
             return *maybeString;
         }
-        auto accessResult = getOptionalAttrField(v, {sOutPath}, pos);
-        if (accessResult.error) throwTypeError(pos, "cannot coerce a set to a string");
-        return coerceToString(pos, *accessResult.value, context, coerceMore, copyToStore);
+        auto i = v.attrs->find(sOutPath);
+        if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
+        return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
     }
 
     if (v.type() == nExternal)
@@ -2350,13 +2002,6 @@ void EvalState::printStats()
         topObj.attr("nrLookups", nrLookups);
         topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
         topObj.attr("nrFunctionCalls", nrFunctionCalls);
-        {
-            auto cache = topObj.object("evalCache");
-            cache.attr("nrCacheMisses", nrCacheMisses);
-            cache.attr("nrCacheHits", nrCacheHits);
-            cache.attr("nrUncached", nrUncached);
-            cache.attr("nrUncacheable", nrUncacheable);
-        }
 #if HAVE_BOEHMGC
         {
             auto gc = topObj.object("gc");
@@ -2458,19 +2103,5 @@ EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);
 
-std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
-{
-    if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
-        return iter->second;
-
-    if (!(evalSettings.useEvalCache && evalSettings.pureEval))
-        return nullptr;
-    auto thisCache = tree_cache::Cache::tryCreate(
-            cacheKey,
-            symbols
-    );
-    evalCache.insert({cacheKey, thisCache});
-    return thisCache;
-}
 
 }

src/libexpr/eval.hh

@@ -1,14 +1,10 @@
 #pragma once
 
 #include "attr-set.hh"
-#include "context.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
-#include "hash.hh"
-#include "sqlite.hh"
-#include "tree-cache.hh"
 
 #include <map>
 #include <optional>
@@ -18,9 +14,6 @@
 
 namespace nix {
 
-namespace eval_cache {
-    class EvalCache;
-}
 
 class Store;
 class EvalState;
@@ -123,8 +116,6 @@ private:
 #endif
     FileEvalCache fileEvalCache;
 
-    std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
-
     SearchPath searchPath;
 
     std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
@@ -140,9 +131,6 @@ public:
     EvalState(const Strings & _searchPath, ref<Store> store);
     ~EvalState();
 
-    std::shared_ptr<eval_cache::EvalCache> openCache(Hash, std::function<Value *()> rootLoader);
-    std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
-
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
@@ -315,26 +303,6 @@ public:
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, Pos * pos);
 
-    struct AttrAccessError {
-        const Symbol attrName;
-
-        // To distinguish between a missing field in an attribute set
-        // and an access to something that's not an attribute set
-        std::optional<Value* > illTypedValue;
-    };
-    struct AttrAccesResult {
-        std::optional<AttrAccessError> error;
-        const Pos * pos;
-        Value* value;
-    };
-    AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-    AttrAccesResult getOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
-    AttrAccesResult lazyGetOptionalAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
-    void getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-    Value* getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos);
-
-    std::vector<Symbol> listAttrFields(Value & attrs, const Pos & pos);
-
     void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
@@ -342,8 +310,6 @@ public:
 
     void realiseContext(const PathSet & context);
 
-    void updateCacheStats(ValueCache::CacheResult);
-
 private:
 
     unsigned long nrEnvs = 0;
@@ -357,10 +323,6 @@ private:
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
-    unsigned long nrCacheHits = 0;
-    unsigned long nrCacheMisses = 0;
-    unsigned long nrUncacheable = 0;
-    unsigned long nrUncached = 0;
 
     bool countCalls;
 

src/libexpr/flake/flake.cc

@@ -617,13 +617,9 @@ void callFlake(EvalState & state,
             , "/"), **vCallFlake);
     }
 
-    vTmp1->mkApp(*vCallFlake, vLocks);
-    vTmp2->mkApp(vTmp1, vRootSrc);
-    vRes.mkApp(vTmp2, vRootSubdir);
-    auto fingerprint = lockedFlake.getFingerprint();
-    auto evalCache = state.openTreeCache(fingerprint);
-    auto cacheRoot = evalCache ? evalCache->getRoot() : nullptr;
-    vRes.setCache(ValueCache(cacheRoot));
+    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -662,19 +658,4 @@ Fingerprint LockedFlake::getFingerprint() const
 
 Flake::~Flake() { }
 
-Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake)
-{
-    /* For testing whether the evaluation cache is
-        complete. */
-    if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
-        throw Error("not everything is cached, but evaluation is not allowed");
-
-    auto vFlake = state.allocValue();
-    flake::callFlake(state, lockedFlake, *vFlake);
-
-    auto aOutputs = state.getAttrField(*vFlake, {state.symbols.create("outputs")}, noPos);
-
-    return aOutputs;
-}
-
 }

src/libexpr/flake/flake.hh

@@ -6,6 +6,7 @@
 #include "value.hh"
 
 namespace nix {
+
 class EvalState;
 
 namespace fetchers { struct Tree; }
@@ -138,5 +139,4 @@ void emitTreeAttrs(
     const fetchers::Input & input,
     Value & v, bool emptyRevFallback = false);
 
-Value* getFlakeValue(EvalState & state, const flake::LockedFlake lockedFlake);
 }

src/libexpr/get-drvs.cc

@@ -10,14 +10,14 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, const string & attrPath, std::optional<Value> attrs)
+DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
     : state(&state), attrs(attrs), attrPath(attrPath)
 {
 }
 
 
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
-    : state(&state), attrs(std::nullopt), attrPath("")
+    : state(&state), attrs(nullptr), attrPath("")
 {
     auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
 
@@ -46,28 +46,12 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 }
 
 
-std::optional<Value> DrvInfo::queryOptionalAttr(const Symbol attrName) const
-{
-    // Erk
-    Value * mutableAttrs = const_cast<Value*>(&attrs.value());
-    auto accessResult = state->getOptionalAttrField(*mutableAttrs, {attrName}, noPos);
-    return accessResult.error ? std::nullopt : std::optional{*accessResult.value};
-}
-
-Value DrvInfo::queryAttr(const Symbol attrName) const
-{
-    Value res;
-    // Erk
-    Value * mutableAttrs = const_cast<Value*>(&attrs.value());
-    state->getAttrField(*mutableAttrs, {attrName}, noPos, res);
-    return res;
-}
-
 string DrvInfo::queryName() const
 {
     if (name == "" && attrs) {
-        Value nameVal = queryAttr(state->sName);
-        name = state->forceStringNoCtx(nameVal);
+        auto i = attrs->find(state->sName);
+        if (i == attrs->end()) throw TypeError("derivation name missing");
+        name = state->forceStringNoCtx(*i->value);
     }
     return name;
 }
@@ -76,8 +60,8 @@ string DrvInfo::queryName() const
 string DrvInfo::querySystem() const
 {
     if (system == "" && attrs) {
-        auto maybeSystemVal = queryOptionalAttr(state->sSystem);
-        system = maybeSystemVal ? state->forceStringNoCtx(*maybeSystemVal) : "unknown";
+        auto i = attrs->find(state->sSystem);
+        system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos);
     }
     return system;
 }
@@ -86,10 +70,9 @@ string DrvInfo::querySystem() const
 string DrvInfo::queryDrvPath() const
 {
     if (drvPath == "" && attrs) {
-        auto drvPathVal = queryOptionalAttr(state->sDrvPath);
+        Bindings::iterator i = attrs->find(state->sDrvPath);
         PathSet context;
-        if (drvPathVal)
-            drvPath = state->coerceToPath(noPos, *drvPathVal, context);
+        drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
     }
     return drvPath;
 }
@@ -98,10 +81,10 @@ string DrvInfo::queryDrvPath() const
 string DrvInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
-        auto val = queryOptionalAttr(state->sOutPath);
+        Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
-        if (val)
-            outPath = state->coerceToPath(noPos, *val, context);
+        if (i != attrs->end())
+            outPath = state->coerceToPath(*i->pos, *i->value, context);
     }
     if (!outPath)
         throw UnimplementedError("CA derivations are not yet supported");
@@ -113,23 +96,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
-        std::optional<Value> outputsList;
-        if (attrs && (outputsList = queryOptionalAttr(state->sOutputs))) {
-            state->forceList(*outputsList);
+        Bindings::iterator i;
+        if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) {
+            state->forceList(*i->value, *i->pos);
 
             /* For each output... */
-            for (unsigned int j = 0; j < outputsList->listSize(); ++j) {
+            for (unsigned int j = 0; j < i->value->listSize(); ++j) {
                 /* Evaluate the corresponding set. */
-                string name = state->forceStringNoCtx(*outputsList->listElems()[j]);
-                auto out = queryOptionalAttr(state->symbols.create(name));
-                if (!out) continue;
-                state->forceAttrs(*out);
+                string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
+                Bindings::iterator out = attrs->find(state->symbols.create(name));
+                if (out == attrs->end()) continue; // FIXME: throw error?
+                state->forceAttrs(*out->value);
 
                 /* And evaluate its ‘outPath’ attribute. */
-                Value outPath;
-                state->getAttrField(*out, {state->sOutPath}, noPos, outPath);
+                Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
+                if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
                 PathSet context;
-                outputs[name] = state->coerceToPath(noPos, outPath, context);
+                outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
             }
         } else
             outputs["out"] = queryOutPath();
@@ -157,8 +140,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 string DrvInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
-        auto val = queryOptionalAttr(state->sOutputName);
-        outputName = val ? state->forceStringNoCtx(*val) : "";
+        Bindings::iterator i = attrs->find(state->sOutputName);
+        outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : "";
     }
     return outputName;
 }
@@ -168,10 +151,10 @@ Bindings * DrvInfo::getMeta()
 {
     if (meta) return meta;
     if (!attrs) return 0;
-    auto val = queryOptionalAttr(state->sMeta);
-    if (!val) return 0;
-    state->forceAttrs(*val);
-    meta = val->attrs;
+    Bindings::iterator a = attrs->find(state->sMeta);
+    if (a == attrs->end()) return 0;
+    state->forceAttrs(*a->value, *a->pos);
+    meta = a->value->attrs;
     return meta;
 }
 
@@ -302,7 +285,7 @@ static bool getDerivation(EvalState & state, Value & v,
            derivation {...}; y = x;}'. */
         if (!done.insert(v.attrs).second) return false;
 
-        DrvInfo drv(state, attrPath, v);
+        DrvInfo drv(state, attrPath, v.attrs);
 
         drv.queryName();
 

src/libexpr/get-drvs.hh

@@ -26,8 +26,7 @@ private:
 
     bool failed = false; // set if we get an AssertionError
 
-    std::optional<Value> attrs;
-    Bindings * meta = nullptr;
+    Bindings * attrs = nullptr, * meta = nullptr;
 
     Bindings * getMeta();
 
@@ -37,12 +36,9 @@ public:
     string attrPath; /* path towards the derivation */
 
     DrvInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, const string & attrPath, std::optional<Value> attrs);
+    DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
     DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
-    Value queryAttr(const Symbol attrName) const;
-    std::optional<Value> queryOptionalAttr(const Symbol attrName) const;
-
     string queryName() const;
     string querySystem() const;
     string queryDrvPath() const;

src/libexpr/nixexpr.hh

@@ -166,10 +166,6 @@ struct ExprSelect : Expr
     AttrPath attrPath;
     ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
     ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
-    ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & names) : pos(pos), e(e), def(0) {
-        for (auto & name : names)
-            attrPath.push_back(AttrName(name));
-    };
     COMMON_METHODS
 };
 

src/libexpr/primops.cc

@@ -410,7 +410,7 @@ static RegisterPrimOp primop_isNull({
       Return `true` if *e* evaluates to `null`, and `false` otherwise.
 
       > **Warning**
-      >
+      > 
       > This function is *deprecated*; just write `e == null` instead.
     )",
     .fun = prim_isNull,
@@ -813,19 +813,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 {
     state.forceAttrs(*args[0], pos);
 
-    Value & arg = *(args[0]);
-
-    auto getOptionalAttr= [&](Symbol attrName) -> std::optional<Value> {
-        auto accessResult = state.getOptionalAttrField(arg, {attrName}, pos);
-        return accessResult.error ? std::nullopt : std::optional{*accessResult.value};
-    };
-
     /* Figure out the name first (for stack backtraces). */
-    state.getAttrField(arg, {state.sName}, pos, v);
+    Bindings::iterator attr = args[0]->attrs->find(state.sName);
+    if (attr == args[0]->attrs->end())
+        throw EvalError({
+            .msg = hintfmt("required attribute 'name' missing"),
+            .errPos = pos
+        });
     string drvName;
-    auto posDrvName = pos; // FIXME: Should be tho position of the `name` attribute
+    Pos & posDrvName(*attr->pos);
     try {
-        drvName = state.forceStringNoCtx(v, pos);
+        drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
         e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
         throw;
@@ -834,15 +832,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     /* Check whether attributes should be passed as a JSON file. */
     std::ostringstream jsonBuf;
     std::unique_ptr<JSONObject> jsonObject;
-    if (auto maybeRawObj = getOptionalAttr(state.sStructuredAttrs)) {
-        if (state.forceBool(*maybeRawObj, pos))
+    attr = args[0]->attrs->find(state.sStructuredAttrs);
+    if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos))
         jsonObject = std::make_unique<JSONObject>(jsonBuf);
-    }
 
     /* Check whether null attributes should be ignored. */
     bool ignoreNulls = false;
-    if (auto maybeRawIgnoreNull = getOptionalAttr(state.sIgnoreNulls))
-        ignoreNulls = state.forceBool(*maybeRawIgnoreNull, pos);
+    attr = args[0]->attrs->find(state.sIgnoreNulls);
+    if (attr != args[0]->attrs->end())
+        ignoreNulls = state.forceBool(*attr->value, pos);
 
     /* Build the derivation expression by processing the attributes. */
     Derivation drv;
@@ -858,9 +856,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     StringSet outputs;
     outputs.insert("out");
 
-    for (auto & currentArg : args[0]->attrs->lexicographicOrder()) {
-        if (currentArg->name == state.sIgnoreNulls) continue;
-        const string & key = currentArg->name;
+    for (auto & i : args[0]->attrs->lexicographicOrder()) {
+        if (i->name == state.sIgnoreNulls) continue;
+        const string & key = i->name;
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string & s) {
@@ -902,26 +900,22 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
         try {
 
-            Value argValue;
-            state.getAttrField(arg, {currentArg->name}, pos, argValue);
-
-
             if (ignoreNulls) {
-                state.forceValue(argValue, pos);
-                if (argValue.type() == nNull) continue;
+                state.forceValue(*i->value, pos);
+                if (i->value->type() == nNull) continue;
             }
 
-            if (currentArg->name == state.sContentAddressed) {
+            if (i->name == state.sContentAddressed) {
                 settings.requireExperimentalFeature("ca-derivations");
-                contentAddressed = state.forceBool(argValue, pos);
+                contentAddressed = state.forceBool(*i->value, pos);
             }
 
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
-            else if (currentArg->name == state.sArgs) {
-                state.forceList(argValue, pos);
-                for (unsigned int n = 0; n < argValue.listSize(); ++n) {
-                    string s = state.coerceToString(posDrvName, *argValue.listElems()[n], context, true);
+            else if (i->name == state.sArgs) {
+                state.forceList(*i->value, pos);
+                for (unsigned int n = 0; n < i->value->listSize(); ++n) {
+                    string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
                     drv.args.push_back(s);
                 }
             }
@@ -932,39 +926,39 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
                 if (jsonObject) {
 
-                    if (currentArg->name == state.sStructuredAttrs) continue;
+                    if (i->name == state.sStructuredAttrs) continue;
 
                     auto placeholder(jsonObject->placeholder(key));
-                    printValueAsJSON(state, true, argValue, placeholder, context);
+                    printValueAsJSON(state, true, *i->value, placeholder, context);
 
-                    if (currentArg->name == state.sBuilder)
-                        drv.builder = state.forceString(argValue, context, posDrvName);
-                    else if (currentArg->name == state.sSystem)
-                        drv.platform = state.forceStringNoCtx(argValue, posDrvName);
-                    else if (currentArg->name == state.sOutputHash)
-                        outputHash = state.forceStringNoCtx(argValue, posDrvName);
-                    else if (currentArg->name == state.sOutputHashAlgo)
-                        outputHashAlgo = state.forceStringNoCtx(argValue, posDrvName);
-                    else if (currentArg->name == state.sOutputHashMode)
-                        handleHashMode(state.forceStringNoCtx(argValue, posDrvName));
-                    else if (currentArg->name == state.sOutputs) {
+                    if (i->name == state.sBuilder)
+                        drv.builder = state.forceString(*i->value, context, posDrvName);
+                    else if (i->name == state.sSystem)
+                        drv.platform = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sOutputHash)
+                        outputHash = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sOutputHashAlgo)
+                        outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName);
+                    else if (i->name == state.sOutputHashMode)
+                        handleHashMode(state.forceStringNoCtx(*i->value, posDrvName));
+                    else if (i->name == state.sOutputs) {
                         /* Require ‘outputs’ to be a list of strings. */
-                        state.forceList(argValue, posDrvName);
+                        state.forceList(*i->value, posDrvName);
                         Strings ss;
-                        for (unsigned int n = 0; n < argValue.listSize(); ++n)
-                            ss.emplace_back(state.forceStringNoCtx(*argValue.listElems()[n], posDrvName));
+                        for (unsigned int n = 0; n < i->value->listSize(); ++n)
+                            ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName));
                         handleOutputs(ss);
                     }
 
                 } else {
-                    auto s = state.coerceToString(posDrvName, argValue, context, true);
+                    auto s = state.coerceToString(posDrvName, *i->value, context, true);
                     drv.env.emplace(key, s);
-                    if (currentArg->name == state.sBuilder) drv.builder = s;
-                    else if (currentArg->name == state.sSystem) drv.platform = s;
-                    else if (currentArg->name == state.sOutputHash) outputHash = s;
-                    else if (currentArg->name == state.sOutputHashAlgo) outputHashAlgo = s;
-                    else if (currentArg->name == state.sOutputHashMode) handleHashMode(s);
-                    else if (currentArg->name == state.sOutputs)
+                    if (i->name == state.sBuilder) drv.builder = s;
+                    else if (i->name == state.sSystem) drv.platform = s;
+                    else if (i->name == state.sOutputHash) outputHash = s;
+                    else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s;
+                    else if (i->name == state.sOutputHashMode) handleHashMode(s);
+                    else if (i->name == state.sOutputs)
                         handleOutputs(tokenizeString<Strings>(s));
                 }
 
@@ -1924,26 +1918,26 @@ static RegisterPrimOp primop_path({
       An enrichment of the built-in path type, based on the attributes
       present in *args*. All are optional except `path`:
 
-        - path
+        - path  
           The underlying path.
 
-        - name
+        - name  
           The name of the path when added to the store. This can used to
           reference paths that have nix-illegal characters in their names,
           like `@`.
 
-        - filter
+        - filter  
           A function of the type expected by `builtins.filterSource`,
           with the same semantics.
 
-        - recursive
+        - recursive  
           When `false`, when `path` is added to the store it is with a
           flat hash, rather than a hash of the NAR serialization of the
           file. Thus, `path` must refer to a regular file, not a
           directory. This allows similar behavior to `fetchurl`. Defaults
           to `true`.
 
-        - sha256
+        - sha256  
           When provided, this is the expected hash of the file at the
           path. Evaluation will fail if the hash is incorrect, and
           providing a hash allows `builtins.path` to be used even when the
@@ -1962,14 +1956,13 @@ static RegisterPrimOp primop_path({
    strings. */
 static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    auto attrNames = state.listAttrFields(*args[0], pos);
     state.forceAttrs(*args[0], pos);
 
-    state.mkList(v, attrNames.size());
+    state.mkList(v, args[0]->attrs->size());
 
     size_t n = 0;
-    for (auto & name : attrNames)
-        mkString(*(v.listElems()[n++] = state.allocValue()), name);
+    for (auto & i : *args[0]->attrs)
+        mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
 
     std::sort(v.listElems(), v.listElems() + n,
               [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
@@ -2020,8 +2013,17 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string attr = state.forceStringNoCtx(*args[0], pos);
     state.forceAttrs(*args[1], pos);
-    state.getAttrField(*args[1], {state.symbols.create(attr)}, pos, v);
     // !!! Should we create a symbol here or just do a lookup?
+    Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
+    if (i == args[1]->attrs->end())
+        throw EvalError({
+            .msg = hintfmt("attribute '%1%' missing", attr),
+            .errPos = pos
+        });
+    // !!! add to stack trace?
+    if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
+    state.forceValue(*i->value, pos);
+    v = *i->value;
 }
 
 static RegisterPrimOp primop_getAttr({
@@ -2354,13 +2356,7 @@ static RegisterPrimOp primop_isList({
 
 static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v)
 {
-    auto evalCache = list.getCache();
-    auto atSymbol = state.symbols.create(std::to_string(n));
-    auto cacheResult = evalCache.getValue(*state.store, {atSymbol}, v);
-    if (cacheResult.returnCode == ValueCache::CacheHit) return;
-
     state.forceList(list, pos);
-    evalCache.addListChilds(state.symbols, list.listElems(), list.listSize());
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error({
             .msg = hintfmt("list index %1% is out of bounds", n),
@@ -2428,7 +2424,7 @@ static RegisterPrimOp primop_tail({
       the argument isn’t a list or is an empty list.
 
       > **Warning**
-      >
+      > 
       > This function should generally be avoided since it's inefficient:
       > unlike Haskell's `tail`, it takes O(n) time, so recursing over a
       > list by repeatedly calling `tail` takes O(n^2) time.

src/libexpr/tree-cache.cc

@@ -1,390 +0,0 @@
-#include "tree-cache.hh"
-#include "sqlite.hh"
-#include "store-api.hh"
-#include "context.hh"
-
-namespace nix::tree_cache {
-
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    id          integer primary key autoincrement not null,
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    context     text,
-    unique      (parent, name)
-);
-
-create index if not exists IndexByParent on Attributes(parent, name);
-)sql";
-
-struct AttrDb
-{
-    std::atomic_bool failed{false};
-
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt updateAttribute;
-        SQLiteStmt insertAttributeWithContext;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->updateAttribute.create(state->db,
-            "update Attributes set type = ?, value = ?, context = ? where id = ?");
-
-        state->insertAttributeWithContext.create(state->db,
-            "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select id, type, value, context from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            if (!failed)
-                state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    template<typename F>
-    AttrId doSQLite(F && fun)
-    {
-        if (failed) return 0;
-        try {
-            return fun();
-        } catch (SQLiteError &) {
-            ignoreException();
-            failed = true;
-            return 0;
-        }
-    }
-
-    /**
-     * Store a leaf of the tree in the db
-     */
-    AttrId addEntry(
-        const AttrKey & key,
-        const AttrValue & value)
-    {
-        return doSQLite([&]()
-        {
-            auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->insertAttributeWithContext.use()
-                (key.first)
-                (key.second)
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                .exec();
-            AttrId rowId = state->db.getLastInsertedRowId();
-            assert(rowId);
-            return rowId;
-        });
-    }
-
-    std::optional<AttrId> getId(const AttrKey& key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return std::nullopt;
-
-        return (AttrType) queryAttribute.getInt(0);
-    }
-
-    AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
-    {
-        debug("cache: miss for the attribute %s", key.second);
-        if (auto existingId = getId(key)) {
-            setValue(*existingId, value);
-            return *existingId;
-        }
-        return addEntry(key, value);
-    }
-
-    void setValue(const AttrId & id, const AttrValue & value)
-    {
-        auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->updateAttribute.use()
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                (id)
-                .exec();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        switch (type) {
-            case AttrType::Attrs: {
-                return {{rowId, attributeSet_t()}};
-            }
-            case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
-                if (!queryAttribute.isNull(3))
-                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
-                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
-            }
-            case AttrType::Bool:
-                return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
-            case AttrType::Int:
-                return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
-            case AttrType::Double:
-                return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
-            case AttrType::Unknown:
-                return {{rowId, unknown_t{}}};
-            case AttrType::Thunk:
-                return {{rowId, thunk_t{}}};
-            case AttrType::Missing:
-                return {{rowId, missing_t{key.second}}};
-            case AttrType::Failed:
-                return {{rowId, failed_t{queryAttribute.getStr(2)}}};
-            default:
-                throw Error("unexpected type in evaluation cache");
-        }
-    }
-
-    std::vector<std::string> getChildren(AttrId parentId)
-    {
-        std::vector<std::string> res;
-        auto state(_state->lock());
-
-        auto queryAttributes(state->queryAttributes.use()(parentId));
-
-        while (queryAttributes.next())
-            res.push_back(queryAttributes.getStr(0));
-
-        return res;
-    }
-};
-
-Cache::Cache(const Hash & useCache,
-        SymbolTable & symbols)
-    : db(std::make_shared<AttrDb>(useCache))
-    , symbols(symbols)
-    , rootSymbol(symbols.create(""))
-{
-}
-
-std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
-{
-    try {
-        return std::make_shared<Cache>(useCache, symbols);
-    } catch (SQLiteError &) {
-        ignoreException();
-        return nullptr;
-    }
-}
-
-void Cache::commit()
-{
-    if (db) {
-        debug("Saving the cache");
-        auto state(db->_state->lock());
-        if (state->txn->active) {
-            state->txn->commit();
-            state->txn.reset();
-            state->txn = std::make_unique<SQLiteTxn>(state->db);
-        }
-    }
-}
-
-Cursor::Ref Cache::getRoot()
-{
-    return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrValue& value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({root->db->setOrUpdate(getKey(), value), value})
-{
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrId & id,
-    const AttrValue & value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({id, value})
-{
-}
-
-
-AttrKey Cursor::getKey()
-{
-    if (!parentId)
-        return {0, root->rootSymbol};
-    return {*parentId, label};
-}
-
-AttrValue Cursor::getCachedValue()
-{
-    return cachedValue.second;
-}
-
-void Cursor::setValue(const AttrValue & v)
-{
-    root->db->setValue(cachedValue.first, v);
-    cachedValue.second = v;
-}
-
-Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
-{
-    Parent parent = {{*this, attrPath}};
-    auto childCursor = new Cursor(
-        root,
-        parent,
-        v
-    );
-    return childCursor;
-}
-
-std::vector<std::string> Cursor::getChildren()
-{
-    return root->db->getChildren(cachedValue.first);
-}
-
-std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
-{
-    auto cursorAtPath = findAlongAttrPath(attrPath);
-    if (cursorAtPath)
-        return cursorAtPath->getChildren();
-    return std::nullopt;
-}
-
-Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
-{
-    auto rawAttr = root->db->getValue({cachedValue.first, name});
-    if (rawAttr) {
-        Parent parent = {{*this, name}};
-        debug("cache: hit for the attribute %s", cachedValue.first);
-        return new Cursor (
-            root, parent, rawAttr->first,
-            rawAttr->second);
-    }
-    if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
-        // If the parent is an attribute set but we're not present in the db,
-        // then we're not a member of this attribute set. So mark as missing
-        return addChild(name, missing_t{name});
-    }
-    return nullptr;
-}
-
-Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
-{
-    auto currentCursor = this;
-    for (auto & currentAccessor : attrPath) {
-        currentCursor = currentCursor->maybeGetAttr(currentAccessor);
-        if (!currentCursor)
-            break;
-        if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
-            break;
-        if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
-            break;
-    }
-    return currentCursor;
-}
-
-const RawValue RawValue::fromVariant(const AttrValue & value)
-{
-    RawValue res;
-    std::visit(overloaded{
-      [&](attributeSet_t x) { res.type = AttrType::Attrs; },
-      [&](string_t x) {
-        res.type = AttrType::String;
-        res.value = x.first;
-        res.context = x.second;
-      },
-      [&](wrapped_basetype<bool> x) {
-        res.type = AttrType::Bool;
-        res.value = x.value ? "1" : "0";
-      },
-      [&](wrapped_basetype<int64_t> x) {
-        res.type = AttrType::Int;
-        res.value = std::to_string(x.value);
-      },
-      [&](wrapped_basetype<double> x) {
-        res.type = AttrType::Double;
-        res.value = std::to_string(x.value);
-      },
-      [&](unknown_t x) { res.type = AttrType::Unknown; },
-      [&](missing_t x) { res.type = AttrType::Missing; },
-      [&](thunk_t x) { res.type = AttrType::Thunk; },
-      [&](failed_t x) {
-          res.type = AttrType::Failed;
-          res.value = x.error;
-      }
-    }, value);
-    return res;
-}
-
-std::string RawValue::serializeContext() const
-{
-    std::string res;
-    for (auto & elt : context) {
-        res.append(encodeContext(elt.second, elt.first));
-        res.push_back(' ');
-    }
-    if (!res.empty())
-        res.pop_back(); // Remove the trailing space
-    return res;
-}
-
-}

src/libexpr/tree-cache.hh

@@ -1,156 +0,0 @@
-/**
- * caching for a tree-like data structure (like Nix values)
- *
- * The cache is an sqlite db whose rows are the nodes of the tree, with a
- * pointer to their parent (except for the root of course)
- */
-
-#pragma once
-
-#include "sync.hh"
-#include "hash.hh"
-#include "symbol-table.hh"
-
-#include <functional>
-#include <variant>
-
-namespace nix::tree_cache {
-
-struct AttrDb;
-class Cursor;
-
-class Cache : public std::enable_shared_from_this<Cache>
-{
-private:
-    friend class Cursor;
-
-    /**
-     * The database holding the cache
-     */
-    std::shared_ptr<AttrDb> db;
-
-    SymbolTable & symbols;
-
-    /**
-     * Distinguished symbol indicating the root of the tree
-     */
-    const Symbol rootSymbol;
-
-public:
-
-    Cache(
-        const Hash & useCache,
-        SymbolTable & symbols
-    );
-
-    static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
-
-    Cursor * getRoot();
-
-    /**
-     * Flush the cache to disk
-     */
-    void commit();
-};
-
-enum AttrType {
-    Unknown = 0,
-    Attrs = 1,
-    String = 2,
-    Bool = 3,
-    Int = 4,
-    Double = 5,
-    Thunk = 6,
-    Missing = 7, // Missing fields of attribute sets
-    Failed = 8,
-};
-
-struct attributeSet_t {};
-struct unknown_t {};
-struct thunk_t {};
-struct failed_t { string error; };
-struct missing_t { Symbol attrName; };
-
-// Putting several different primitive types in an `std::variant` partially
-// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
-// from one to another which breaks the exhaustiveness check.
-// So we wrap them in a trivial class just to force the disambiguation
-template<typename T>
-struct wrapped_basetype{ T value; };
-
-typedef uint64_t AttrId;
-
-typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
-
-typedef std::variant<
-    attributeSet_t,
-    string_t,
-    unknown_t,
-    thunk_t,
-    missing_t,
-    failed_t,
-    wrapped_basetype<bool>,
-    wrapped_basetype<int64_t>,
-    wrapped_basetype<double>
-> AttrValue;
-
-struct RawValue {
-    AttrType type;
-    std::optional<std::string> value;
-    std::vector<std::pair<Path, std::string>> context;
-
-    std::string serializeContext() const;
-
-    static const RawValue fromVariant(const AttrValue&);
-    AttrValue toVariant() const;
-};
-
-/**
- * View inside the cache.
- *
- * A `Cursor` represents a node in the cached tree (be it a leaf or not)
- */
-class Cursor : public std::enable_shared_from_this<Cursor>
-{
-    /**
-     * The overall cache of which this cursor is a view
-     */
-    ref<Cache> root;
-
-    typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
-
-    std::optional<AttrId> parentId;
-    Symbol label;
-
-    std::pair<AttrId, AttrValue> cachedValue;
-
-    /**
-     * Get the identifier for this node in the database
-     */
-    AttrKey getKey();
-
-public:
-
-    using Ref = Cursor*;
-
-    // Create a new cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
-    // Build a cursor from an existing cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
-
-    AttrValue getCachedValue();
-
-    void setValue(const AttrValue & v);
-
-    Ref addChild(const Symbol & attrPath, const AttrValue & v);
-
-    Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
-    Ref maybeGetAttr(const Symbol & attrPath);
-
-
-    std::vector<std::string> getChildren();
-    std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
-};
-
-}

src/libexpr/value.hh

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "symbol-table.hh"
-#include "tree-cache.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -57,51 +56,8 @@ struct Pos;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
-class Store;
 
 
-struct Value;
-class ValueCache {
-    tree_cache::Cursor::Ref rawCache;
-
-public:
-
-    ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
-
-    const static ValueCache empty;
-
-    enum ReturnCode {
-        // The cache result was an attribute set, so we forward it later in the
-        // chain
-        Forward,
-        CacheMiss,
-        CacheHit,
-        UnCacheable,
-        NoCacheKey,
-    };
-
-    struct CacheResult {
-        ReturnCode returnCode;
-
-        // In case the query returns a `missing_t`, the symbol that's missing
-        std::optional<Symbol> lastQueriedSymbolIfMissing;
-    };
-    CacheResult getValue(Store & store, const std::vector<Symbol> & selector, Value & dest);
-
-    ValueCache addChild(const Symbol & attrName, const Value & value);
-    ValueCache addFailedChild(const Symbol & attrName, const Error & error);
-    ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
-    void addAttrSetChilds(Bindings & children);
-    void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
-
-    std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
-    std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
-
-    std::optional<tree_cache::AttrValue> getRawValue();
-
-    ValueCache() : rawCache(nullptr) {}
-};
-
 typedef int64_t NixInt;
 typedef double NixFloat;
 
@@ -157,12 +113,6 @@ friend std::string showType(const Value & v);
 friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
 
 public:
-    /*
-     * An optional evaluation cache (for flakes in particular).
-     * If this is set, then trying to get a value from this attrset will first
-     * try to get it from the cache
-     */
-    ValueCache evalCache;
 
     // Functions needed to distinguish the type
     // These should be removed eventually, by putting the functionality that's
@@ -396,19 +346,7 @@ public:
        non-trivial. */
     bool isTrivial() const;
 
-    std::vector<std::pair<Path, std::string>> getContext() const;
-
-    /*
-     * Set the associated cache view for this value.
-     * This cache will be used to speed-up some operations like accessing
-     * attribute sets elements.
-     */
-    void setCache(ValueCache);
-
-    /*
-     * Get the cache associated with this value, if any.
-     */
-    ValueCache getCache() const;
+    std::vector<std::pair<Path, std::string>> getContext();
 };
 
 

src/libutil/regex.hh

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <boost/regex.hpp>
+
+namespace regex = boost;
+

src/nix-env/nix-env.cc

@@ -374,7 +374,7 @@ static void queryInstSources(EvalState & state,
 
                 std::string name(path.name());
 
-                DrvInfo elem(state, "", std::nullopt);
+                DrvInfo elem(state, "", nullptr);
                 elem.setName(name);
 
                 if (path.isDerivation()) {

src/nix/app.cc

@@ -1,28 +1,26 @@
 #include "installables.hh"
 #include "store-api.hh"
 #include "eval-inline.hh"
+#include "eval-cache.hh"
 #include "names.hh"
 
 namespace nix {
 
 App Installable::toApp(EvalState & state)
 {
-    auto [value, pos, attrPath] = toValue(state);
+    auto [cursor, attrPath] = getCursor(state);
 
-    auto type = state.forceStringNoCtx(*state.getAttrField(*value, {state.sType}, pos));
+    auto type = cursor->getAttr("type")->getString();
 
     if (type == "app") {
-        StringSet context;
-        auto program = state.forceString(*state.getAttrField(*value, {state.symbols.create("program")}, pos));
+        auto [program, context] = cursor->getAttr("program")->getStringWithContext();
 
         if (!state.store->isInStore(program))
             throw Error("app program '%s' is not in the Nix store", program);
 
         std::vector<StorePathWithOutputs> context2;
-        for (auto & rawCtxItem : context) {
-            auto [path, name] = decodeContext(rawCtxItem);
+        for (auto & [path, name] : context)
             context2.push_back({state.store->parseStorePath(path), {name}});
-        }
 
         return App {
             .context = std::move(context2),
@@ -31,10 +29,10 @@ App Installable::toApp(EvalState & state)
     }
 
     else if (type == "derivation") {
-        auto drvPath = state.store->parseStorePath(state.forceString(*state.getAttrField(*value, {state.sDrvPath}, pos)));
-        auto outPath = state.forceString(*state.getAttrField(*value, {state.sOutPath}, pos));
-        auto outputName = state.forceStringNoCtx(*state.getAttrField(*value, {state.sOutputName}, pos));
-        auto name = state.forceStringNoCtx(*state.getAttrField(*value, {state.sName}, pos));
+        auto drvPath = cursor->forceDerivation();
+        auto outPath = cursor->getAttr(state.sOutPath)->getString();
+        auto outputName = cursor->getAttr(state.sOutputName)->getString();
+        auto name = cursor->getAttr(state.sName)->getString();
         return App {
             .context = { { drvPath, {outputName} } },
             .program = outPath + "/bin/" + DrvName(name).name,

src/nix/bundle.cc

@@ -92,7 +92,7 @@ struct CmdBundle : InstallableCommand
         arg->attrs->sort();
 
         auto vRes = evalState->allocValue();
-        evalState->callFunction(*bundler.toValue(*evalState).value, *arg, *vRes, noPos);
+        evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
 
         if (!evalState->isDerivation(*vRes))
             throw Error("the bundler '%s' does not produce a derivation", bundler.what());

src/nix/develop.cc

@@ -6,8 +6,7 @@
 #include "derivations.hh"
 #include "affinity.hh"
 #include "progress-bar.hh"
-
-#include <regex>
+#include "regex.hh"
 
 using namespace nix;
 
@@ -64,42 +63,42 @@ BuildEnvironment readEnvironment(const Path & path)
     static std::string indexedArrayRegex =
         R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
 
-    static std::regex declareRegex(
+    static regex::regex declareRegex(
         "^declare -a?x (" + varNameRegex + ")(=(" +
         dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
 
-    static std::regex varRegex(
+    static regex::regex varRegex(
         "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
 
     /* Note: we distinguish between an indexed and associative array
        using the space before the closing parenthesis. Will
        undoubtedly regret this some day. */
-    static std::regex assocArrayRegex(
+    static regex::regex assocArrayRegex(
         "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
 
-    static std::regex functionRegex(
+    static regex::regex functionRegex(
         "^" + varNameRegex + " \\(\\) *\n");
 
     while (pos != file.end()) {
 
-        std::smatch match;
+        regex::smatch match;
 
-        if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
+        if (regex::regex_search(pos, file.cend(), match, declareRegex, regex::regex_constants::match_continuous)) {
             pos = match[0].second;
             exported.insert(match[1]);
         }
 
-        else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
+        else if (regex::regex_search(pos, file.cend(), match, varRegex, regex::regex_constants::match_continuous)) {
             pos = match[0].second;
             res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
         }
 
-        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
+        else if (regex::regex_search(pos, file.cend(), match, assocArrayRegex, regex::regex_constants::match_continuous)) {
             pos = match[0].second;
             res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
         }
 
-        else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
+        else if (regex::regex_search(pos, file.cend(), match, functionRegex, regex::regex_constants::match_continuous)) {
             res.bashFunctions = std::string(pos, file.cend());
             break;
         }

src/nix/edit.cc

@@ -28,7 +28,7 @@ struct CmdEdit : InstallableCommand
     {
         auto state = getEvalState();
 
-        auto [v, pos, _] = installable->toValue(*state);
+        auto [v, pos] = installable->toValue(*state);
 
         try {
             pos = findDerivationFilename(*state, *v, installable->what());

src/nix/eval.cc

@@ -60,7 +60,7 @@ struct CmdEval : MixJSON, InstallableCommand
 
         auto state = getEvalState();
 
-        auto [v, pos, _] = installable->toValue(*state);
+        auto [v, pos] = installable->toValue(*state);
         PathSet context;
 
         if (apply) {

src/nix/flake.cc

@@ -11,6 +11,7 @@
 #include "fetchers.hh"
 #include "registry.hh"
 #include "json.hh"
+#include "eval-cache.hh"
 
 #include <nlohmann/json.hpp>
 #include <queue>
@@ -599,9 +600,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             Strings{templateName == "" ? "defaultTemplate" : templateName},
             Strings(attrsPathPrefixes), lockFlags);
 
-        auto flakeValue = installable.toValue(*evalState);
-        PathSet context;
-        auto templateDir = evalState->coerceToString(flakeValue.pos, *evalState->getAttrField(*flakeValue.value, {evalState->symbols.create("path")}, flakeValue.pos), context, false, false);
+        auto [cursor, attrPath] = installable.getCursor(*evalState);
+
+        auto templateDir = cursor->getAttr("path")->getString();
 
         assert(store->isInStore(templateDir));
 
@@ -828,9 +829,9 @@ struct CmdFlakeShow : FlakeCommand
         auto state = getEvalState();
         auto flake = std::make_shared<LockedFlake>(lockFlake());
 
-        std::function<void(Value & value, Pos & pos, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
+        std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
 
-        visit = [&](Value & value, Pos & pos, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
+        visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@@ -838,26 +839,21 @@ struct CmdFlakeShow : FlakeCommand
                 auto recurse = [&]()
                 {
                     logger->cout("%s", headerPrefix);
-                    auto attrs = state->listAttrFields(value, pos);
+                    auto attrs = visitor.getAttrs();
                     for (const auto & [i, attr] : enumerate(attrs)) {
                         bool last = i + 1 == attrs.size();
-                        try {
-                            auto value2 = state->getAttrField(value, {attr}, pos);
-                            auto attrPath2(attrPath);
-                            attrPath2.push_back(attr);
-                            visit(*value2, pos, attrPath2,
-                                fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr),
-                                nextPrefix + (last ? treeNull : treeLine));
-                        } catch (EvalError& e) {
-                            if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
-                                throw;
-                        }
+                        auto visitor2 = visitor.getAttr(attr);
+                        auto attrPath2(attrPath);
+                        attrPath2.push_back(attr);
+                        visit(*visitor2, attrPath2,
+                            fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr),
+                            nextPrefix + (last ? treeNull : treeLine));
                     }
                 };
 
                 auto showDerivation = [&]()
                 {
-                    auto name = state->forceStringNoCtx(*state->getAttrField(value, {state->sName}, pos));
+                    auto name = visitor.getAttr(state->sName)->getString();
 
                     /*
                     std::string description;
@@ -899,14 +895,14 @@ struct CmdFlakeShow : FlakeCommand
                     || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages"))
                     )
                 {
-                    if (state->isDerivation(value))
+                    if (visitor.isDerivation())
                         showDerivation();
                     else
                         throw Error("expected a derivation");
                 }
 
                 else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") {
-                    if (state->isDerivation(value))
+                    if (visitor.isDerivation())
                         showDerivation();
                     else
                         recurse();
@@ -918,7 +914,7 @@ struct CmdFlakeShow : FlakeCommand
                     else if (!showLegacy)
                         logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
                     else {
-                        if (state->isDerivation(value))
+                        if (visitor.isDerivation())
                             showDerivation();
                         else if (attrPath.size() <= 2)
                             // FIXME: handle recurseIntoAttrs
@@ -930,8 +926,8 @@ struct CmdFlakeShow : FlakeCommand
                     (attrPath.size() == 2 && attrPath[0] == "defaultApp") ||
                     (attrPath.size() == 3 && attrPath[0] == "apps"))
                 {
-                    auto aType = state->getOptionalAttrField(value, {state->sType}, pos);
-                    if (aType.error || state->forceStringNoCtx(*aType.value) != "app")
+                    auto aType = visitor.maybeGetAttr("type");
+                    if (!aType || aType->getString() != "app")
                         throw EvalError("not an app definition");
                     logger->cout("%s: app", headerPrefix);
                 }
@@ -940,8 +936,7 @@ struct CmdFlakeShow : FlakeCommand
                     (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") ||
                     (attrPath.size() == 2 && attrPath[0] == "templates"))
                 {
-                    /* auto description = visitor.getAttr("description")->getString(); */
-                    auto description = state->forceStringNoCtx(*state->getAttrField(value, {state->sDescription}, pos));
+                    auto description = visitor.getAttr("description")->getString();
                     logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
                 }
 
@@ -959,9 +954,9 @@ struct CmdFlakeShow : FlakeCommand
             }
         };
 
-        auto flakeValue = getFlakeValue(*state, *flake);
+        auto cache = openEvalCache(*state, flake);
 
-        visit(*flakeValue, noPos, {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
+        visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
     }
 };
 

src/nix/local.mk

@@ -18,7 +18,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown -lboost_regex
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \

src/nix/search.cc

@@ -7,6 +7,7 @@
 #include "common-args.hh"
 #include "json.hh"
 #include "shared.hh"
+#include "eval-cache.hh"
 #include "attr-path.hh"
 
 #include <regex>
@@ -80,38 +81,31 @@ struct CmdSearch : InstallableCommand, MixJSON
 
         uint64_t results = 0;
 
-        std::function<void(Value & cursor, const Pos& pos, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
+        std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
 
-        visit = [&](Value & cursor, const Pos& pos, const std::vector<Symbol> & attrPath, bool initialRecurse)
+        visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)
         {
             Activity act(*logger, lvlInfo, actUnknown,
                 fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
             try {
                 auto recurse = [&]()
                 {
-                    for (const auto & attr : evalState->listAttrFields(cursor, pos)) {
-                        Value* cursor2;
-                        try {
-                            auto cursor2Res = evalState->lazyGetOptionalAttrField(cursor, {attr}, pos);
-                            if (cursor2Res.error)
-                                continue; // Shouldn't happen, but who knows
-                            cursor2 = cursor2Res.value;
-                        } catch (EvalError&) {
-                            continue;
-                        }
+                    for (const auto & attr : cursor.getAttrs()) {
+                        auto cursor2 = cursor.getAttr(attr);
                         auto attrPath2(attrPath);
                         attrPath2.push_back(attr);
-                        visit(*cursor2, pos, attrPath2, false);
+                        visit(*cursor2, attrPath2, false);
                     }
                 };
 
-                if (evalState->isDerivation(cursor)) {
+                if (cursor.isDerivation()) {
                     size_t found = 0;
 
-                    DrvName name(evalState->forceStringNoCtx(*evalState->getAttrField(cursor, {evalState->sName}, pos)));
+                    DrvName name(cursor.getAttr("name")->getString());
 
-                    auto descriptionGetRes = evalState->getOptionalAttrField(cursor, {evalState->sMeta, evalState->sDescription}, pos);
-                    auto description = !descriptionGetRes.error ? state->forceStringNoCtx(*descriptionGetRes.value) : "";
+                    auto aMeta = cursor.maybeGetAttr("meta");
+                    auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
+                    auto description = aDescription ? aDescription->getString() : "";
                     std::replace(description.begin(), description.end(), '\n', ' ');
                     auto attrPath2 = concatStringsSep(".", attrPath);
 
@@ -160,8 +154,8 @@ struct CmdSearch : InstallableCommand, MixJSON
                     recurse();
 
                 else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
-                    auto recurseForDrvGetRes = evalState->getOptionalAttrField(cursor, {evalState->sRecurseForDerivations}, pos);
-                    if (!recurseForDrvGetRes.error && evalState->forceBool(*recurseForDrvGetRes.value, pos))
+                    auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
+                    if (attr && attr->getBool())
                         recurse();
                 }
 
@@ -171,8 +165,8 @@ struct CmdSearch : InstallableCommand, MixJSON
             }
         };
 
-        for (auto & [cursor, pos, prefix] : installable->toValues(*state))
-            visit(*cursor, pos, parseAttrPath(*state, prefix), true);
+        for (auto & [cursor, prefix] : installable->getCursors(*state))
+            visit(*cursor, parseAttrPath(*state, prefix), true);
 
         if (!json && !results)
             throw Error("no results for the given search term(s)!");

tests/plugins/local.mk

@@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1
 
 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
 
-libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr -I src/libstore
+libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr