Compare commits

..
Author SHA1 Message Date
regnat ea792bcdc8 Make nix search lazier
Only 4 evals left \o/
2021-06-10 12:26:32 +02:00
regnat 71650c83c6 Make the error cleaner when getFields is misused 2021-06-10 12:26:32 +02:00
regnat d51deeac1c Remove the obsolete CmdSearch::visit function 2021-06-10 12:26:32 +02:00
regnat 1f8541258c Make getFields just return the name of the fields
Makes it much easier to deal with non-evaluating stuff
2021-06-10 12:26:32 +02:00
regnat 3a9753132e Make the derivation check more lazy 2021-06-10 08:09:25 +02:00
regnat 2324ee4891 Add a ugly hack to delay errors in getFields 2021-06-10 08:09:25 +02:00
regnat 28c1f8800b Don’t list known absent attributes in listChildren 2021-06-10 08:09:25 +02:00
regnat af775bdcf9 Make lazyGetAttrField return something more informative 2021-06-10 08:09:25 +02:00
regnat 8d2be51d19 Cache the evaluation errors
Doesn’t seem to make much of a difference on `nix search`, but we’ll
need it at some point
2021-06-10 08:09:25 +02:00
regnat 9102508f33 Use the cache in nix search
Not optimal atm, possibly because we don’t cache the evaluation failures
2021-06-10 08:09:25 +02:00
regnat a6aaf81103 [TMP]: Disable the tests
To allow the benchmarks to run even when something’s broken
2021-06-10 08:09:25 +02:00
regnat c7a232e200 Be even lazier for the build evaluation
Now we can not evaluate anything \o/
2021-06-08 16:05:51 +02:00
regnat 30d14b772f Evaluate more lazily in findAlongAttrPath 2021-06-04 16:53:59 +02:00
regnat fbaee9b8fb Add a “cached thunk” value type 2021-06-04 16:45:22 +02:00
regnat 753730c410 Optimize the nix build caching 2021-06-04 15:21:03 +02:00
regnat c116e6e837 Another attempt at caching nix build
A different tradeof set probably. I guess we could unify both
2021-06-04 15:16:28 +02:00
regnat 3e261410bc First attempt at caching the evaluation in nix build 2021-06-04 11:39:29 +02:00
regnat ffec547ebc Add some benchmarks for the actual caching 2021-06-03 16:41:39 +02:00
regnat 512afd8b7a Use the cache in the outer shell
Make sure that we don’t discard it before entering the evaluator proper
2021-06-03 15:29:43 +02:00
regnat 69505c84e1 Commit the cache at the end of the evaluation
Otherwise the db is never properly filled
2021-06-03 15:29:12 +02:00
regnat af5c323e93 Make the root symbol more telling in the DB
I spent a few minutes trying to understand why I had a field with an
empty symbol, until I realise that it’s because that was the symbol for
the root element.
Now that shouldn’t happen anymore
2021-06-03 15:28:04 +02:00
regnat 9053ac0693 use the setEvalCache function at the flake root
Probably doesn’t change much, but much cleaner (and robust)
2021-06-03 15:27:33 +02:00
regnat b39ab10749 Properly fill the cache in case of a miss/forward 2021-06-03 15:27:03 +02:00
regnat 8787218c7c (maybe) Actually cache things 2021-06-03 13:06:01 +02:00
regnat 89951cf7fb Extract a Value method to set the eval cache 2021-06-03 12:26:22 +02:00
regnat 45a28ed36f Extract a Value method to get the eval cache 2021-06-03 12:26:02 +02:00
regnat 6ec852e7f0 Simplify the forcing of nested records 2021-06-03 12:11:51 +02:00
regnat 7c718646cb Add some stats for the evaluation caching 2021-06-03 12:11:51 +02:00
regnat 8d95e1f299 Set the cache when opening a flake 2021-06-03 12:11:51 +02:00
regnat 6396416dfa Query the eval cache
(It’s still always empty)
2021-06-03 12:11:51 +02:00
regnat 44f390ed48 Re-split the function, but without the exception 2021-06-03 12:11:51 +02:00
regnat 4ca1a0b864 Try removing the exception 2021-06-03 12:11:51 +02:00
regnat 2021b1d8d1 [TMP] Try inlining the getAttrField function
See what’s making things slow
2021-06-03 12:11:51 +02:00
regnat b012852cb2 Split a standalone function for accessing an attribute set field 2021-06-03 12:11:51 +02:00
regnat 074e0678bd Add an (always empty) eval cache to the attr sets 2021-06-03 12:11:51 +02:00
regnat 891390d76f [TMP]: Add some benchmarking tools
Not intended to be merged (at least definitely not as it is)
2021-06-03 12:11:51 +02:00
69 changed files with 1495 additions and 972 deletions

No files matched your search

+2 -1
View File
@@ -20,7 +20,8 @@ jobs:
name: '${{ env.CACHIX_NAME }}' name: '${{ env.CACHIX_NAME }}'
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)') #- run: nix flake check
- run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
check_cachix: check_cachix:
name: Cachix secret present for installer tests name: Cachix secret present for installer tests
runs-on: ubuntu-latest runs-on: ubuntu-latest
Executable
+53
View File
@@ -0,0 +1,53 @@
#!/bin/sh
set -euo pipefail
set -x
callNix () {
nix \
--experimental-features "nix-command flakes" \
--store /tmp/nix \
"$@"
}
callBuild () {
callNix \
eval --impure --file "$THINGTOBENCH" drvPath \
"$@"
}
getCompletions () {
NIX_GET_COMPLETIONS=6 callNix build "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31#firef" "$@"
}
runSearch () {
callNix search "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31" firefox "$@"
}
noCache () {
"$@" --option eval-cache false
}
coldCache () {
if [[ -e ~/.cache/nix/eval-cache-v2 || -e ~/.cache/nix/eval-cache-v3 ]]; then
echo "Error: The cache should be clean"
exit 1
fi
"$@"
}
run_all () {
mkdir -p "$out"
NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$out/eval-stats.json bash $0 noCache callBuild
hyperfine \
--warmup 2 \
--export-csv "$out/result.csv" \
--export-json "$out/result.json" \
--export-markdown "$out/result.md" \
--style basic \
--prepare '' "bash $0 noCache callBuild" \
--prepare 'rm -rf ~/.cache/nix/' "bash $0 coldCache callBuild" \
--prepare '' "bash $0 callBuild"
}
"$@"
+2 -2
View File
@@ -65,7 +65,7 @@ AC_SYS_LARGEFILE
AC_STRUCT_DIRENT_D_TYPE AC_STRUCT_DIRENT_D_TYPE
if test "$sys_name" = sunos; then if test "$sys_name" = sunos; then
# Solaris requires -lsocket -lnsl for network functions # Solaris requires -lsocket -lnsl for network functions
LDFLAGS="-lsocket -lnsl $LDFLAGS" LIBS="-lsocket -lnsl $LIBS"
fi fi
@@ -150,7 +150,7 @@ int main() {
}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
LDFLAGS="-latomic $LDFLAGS" LIBS="-latomic $LIBS"
fi fi
PKG_PROG_PKG_CONFIG PKG_PROG_PKG_CONFIG
+19 -23
View File
@@ -10,39 +10,35 @@ Most Nix commands interpret the following environment variables:
A colon-separated list of directories used to look up Nix A colon-separated list of directories used to look up Nix
expressions enclosed in angle brackets (i.e., `<path>`). For expressions enclosed in angle brackets (i.e., `<path>`). For
instance, the value instance, the value
/home/eelco/Dev:/etc/nixos /home/eelco/Dev:/etc/nixos
will cause Nix to look for paths relative to `/home/eelco/Dev` and will cause Nix to look for paths relative to `/home/eelco/Dev` and
`/etc/nixos`, in this order. It is also possible to match paths `/etc/nixos`, in this order. It is also possible to match paths
against a prefix. For example, the value against a prefix. For example, the value
nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
will cause Nix to search for `<nixpkgs/path>` in will cause Nix to search for `<nixpkgs/path>` in
`/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
If a path in the Nix search path starts with `http://` or If a path in the Nix search path starts with `http://` or
`https://`, it is interpreted as the URL of a tarball that will be `https://`, it is interpreted as the URL of a tarball that will be
downloaded and unpacked to a temporary location. The tarball must downloaded and unpacked to a temporary location. The tarball must
consist of a single top-level directory. For example, setting consist of a single top-level directory. For example, setting
`NIX_PATH` to `NIX_PATH` to
nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
tells Nix to download and use the current contents of the tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
`master` branch in the `nixpkgs` repository. channel.
The URLs of the tarballs from the official nixos.org channels (see A following shorthand can be used to refer to the official channels:
[the manual for `nix-channel`](nix-channel.md)) can be abbreviated
as `channel:<channel-name>`. For instance, the following two nixpkgs=channel:nixos-15.09
values of `NIX_PATH` are equivalent:
The search path can be extended using the `-I` option, which takes
nixpkgs=channel:nixos-21.05 precedence over `NIX_PATH`.
nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
The Nix search path can also be extended using the `-I` option to
many Nix commands, which takes precedence over `NIX_PATH`.
- `NIX_IGNORE_SYMLINK_STORE`\ - `NIX_IGNORE_SYMLINK_STORE`\
Normally, the Nix store directory (typically `/nix/store`) is not Normally, the Nix store directory (typically `/nix/store`) is not
@@ -54,7 +50,7 @@ Most Nix commands interpret the following environment variables:
builds are deployed to machines where `/nix/store` resolves builds are deployed to machines where `/nix/store` resolves
differently. If you are sure that you’re not going to do that, you differently. If you are sure that you’re not going to do that, you
can set `NIX_IGNORE_SYMLINK_STORE` to `1`. can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
Note that if you’re symlinking the Nix store so that you can put it Note that if you’re symlinking the Nix store so that you can put it
on another file system than the root file system, on Linux you’re on another file system than the root file system, on Linux you’re
better off using `bind` mount points, e.g., better off using `bind` mount points, e.g.,
@@ -63,7 +59,7 @@ Most Nix commands interpret the following environment variables:
$ mkdir /nix $ mkdir /nix
$ mount -o bind /mnt/otherdisk/nix /nix $ mount -o bind /mnt/otherdisk/nix /nix
``` ```
Consult the mount 8 manual page for details. Consult the mount 8 manual page for details.
- `NIX_STORE_DIR`\ - `NIX_STORE_DIR`\
Generated
+3 -3
View File
@@ -19,11 +19,11 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1624862269, "lastModified": 1622593737,
"narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=", "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "f77036342e2b690c61c97202bf48f2ce13acc022", "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e",
"type": "github" "type": "github"
}, },
"original": { "original": {
+10 -4
View File
@@ -229,7 +229,7 @@
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
''; '';
doInstallCheck = true; doInstallCheck = false;
installCheckFlags = "sysconfdir=$(out)/etc"; installCheckFlags = "sysconfdir=$(out)/etc";
separateDebugInfo = true; separateDebugInfo = true;
@@ -256,8 +256,7 @@
boost boost
nlohmann_json nlohmann_json
] ]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
configureFlags = '' configureFlags = ''
--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -295,6 +294,13 @@
''; '';
}; };
nix-benchmarks = prev.runCommandNoCC "nix-benchmarks" {
buildInputs = [ final.nix prev.hyperfine ];
THINGTOBENCH = ./thingToBench.nix;
NIX_PATH="nixpkgs=${prev.path}";
} ''
bash ${./benchmark.sh} run_all
'';
}; };
hydraJobs = { hydraJobs = {
@@ -486,7 +492,7 @@
}); });
packages = forAllSystems (system: { packages = forAllSystems (system: {
inherit (nixpkgsFor.${system}) nix; inherit (nixpkgsFor.${system}) nix nix-benchmarks;
} // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
nix-static = let nix-static = let
nixpkgs = nixpkgsFor.${system}.pkgsStatic; nixpkgs = nixpkgsFor.${system}.pkgsStatic;
+1 -3
View File
@@ -1,5 +1,3 @@
#compdef nix
function _nix() { function _nix() {
local ifs_bk="$IFS" local ifs_bk="$IFS"
local input=("${(Q)words[@]}") local input=("${(Q)words[@]}")
@@ -20,4 +18,4 @@ function _nix() {
_describe 'nix' suggestions _describe 'nix' suggestions
} }
_nix "$@" compdef _nix nix
+1 -10
View File
@@ -277,16 +277,7 @@ connected:
auto drv = store->readDerivation(*drvPath); auto drv = store->readDerivation(*drvPath);
auto outputHashes = staticOutputHashes(*store, drv); auto outputHashes = staticOutputHashes(*store, drv);
drv.inputSrcs = store->parseStorePathSet(inputs);
// Hijack the inputs paths of the derivation to include all the paths
// that come from the `inputDrvs` set.
// We don’t do that for the derivations whose `inputDrvs` is empty
// because
// 1. It’s not needed
// 2. Changing the `inputSrcs` set changes the associated output ids,
// which break CA derivations
if (!drv.inputDrvs.empty())
drv.inputSrcs = store->parseStorePathSet(inputs);
auto result = sshStore->buildDerivation(*drvPath, drv); auto result = sshStore->buildDerivation(*drvPath, drv);
+48 -36
View File
@@ -302,6 +302,14 @@ Installable::getCursors(EvalState & state)
return {{evalCache->getRoot(), ""}}; return {{evalCache->getRoot(), ""}};
} }
std::pair<Value*, Pos> Installable::toValue(EvalState & state)
{
auto values = toValues(state);
if (values.empty())
throw Error("cannot find flake attribute '%s'", what());
return {std::get<0>(values[0]), std::get<1>(values[0])};
}
std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
Installable::getCursor(EvalState & state) Installable::getCursor(EvalState & state)
{ {
@@ -378,11 +386,11 @@ struct InstallableAttrPath : InstallableValue
std::string what() override { return attrPath; } std::string what() override { return attrPath; }
std::pair<Value *, Pos> toValue(EvalState & state) override std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override
{ {
auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
state.forceValue(*vRes); state.forceValue(*vRes);
return {vRes, pos}; return {{vRes, pos, attrPath}};
} }
virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -428,12 +436,10 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
callFlake(state, lockedFlake, *vFlake); callFlake(state, lockedFlake, *vFlake);
auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); auto vRes = state.allocValue();
assert(aOutputs); auto gotField = state.lazyGetAttrField(*vFlake, {state.symbols.create("outputs")}, noPos, *vRes);
assert(gotField != EvalState::LazyValueType::Missing);
state.forceValue(*aOutputs->value); return vRes;
return aOutputs->value;
} }
ref<eval_cache::EvalCache> openEvalCache( ref<eval_cache::EvalCache> openEvalCache(
@@ -500,25 +506,34 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto root = cache->getRoot(); auto root = cache->getRoot();
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
auto attr = root->findAlongAttrPath( auto emptyArgs = state->allocBindings(0);
parseAttrPath(*state, attrPath), try {
true auto [drvValue, pos] = findAlongAttrPath(
); *state,
attrPath,
*emptyArgs,
*getFlakeOutputs(*state, *lockedFlake)
);
Value * v = state->allocValue();
if (!state->getAttrField(*drvValue, {state->sDrvPath}, pos, *v))
break;
auto drvPath = state->forceString(*v);
if (!state->getAttrField(*drvValue, {state->sOutPath}, pos, *v))
break;
auto outPath = state->forceString(*v);
if (!state->getAttrField(*drvValue, {state->sOutputName}, pos, *v))
break;
auto outputName = state->forceString(*v);
if (!attr) continue; auto drvInfo = DerivationInfo{
state->store->parseStorePath(drvPath),
state->store->maybeParseStorePath(outPath),
outputName
};
if (!attr->isDerivation()) return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
throw Error("flake output attribute '%s' is not a derivation", attrPath); } catch (AttrPathNotFound & e) {
}
auto drvPath = attr->forceDerivation();
auto drvInfo = DerivationInfo{
std::move(drvPath),
state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
};
return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
} }
throw Error("flake '%s' does not provide attribute %s", throw Error("flake '%s' does not provide attribute %s",
@@ -532,25 +547,22 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
return res; return res;
} }
std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) std::vector<std::tuple<Value *, Pos, std::string>>
InstallableFlake::toValues(EvalState & state)
{ {
auto lockedFlake = getLockedFlake(); auto vOutputs = getFlakeOutputs(state, *getLockedFlake());
auto vOutputs = getFlakeOutputs(state, *lockedFlake);
auto emptyArgs = state.allocBindings(0); auto emptyArgs = state.allocBindings(0);
std::vector<std::tuple<Value *, Pos, std::string>> res;
for (auto & attrPath : getActualAttrPaths()) { for (auto & attrPath : getActualAttrPaths()) {
try { try {
auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
state.forceValue(*v); res.push_back({v, pos, attrPath});
return {v, pos}; } catch (Error &) {}
} catch (AttrPathNotFound & e) {
}
} }
throw Error("flake '%s' does not provide attribute %s", return res;
flakeRef, showAttrPaths(getActualAttrPaths()));
} }
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+3 -2
View File
@@ -41,10 +41,11 @@ struct Installable
UnresolvedApp toApp(EvalState & state); UnresolvedApp toApp(EvalState & state);
virtual std::pair<Value *, Pos> toValue(EvalState & state) virtual std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state)
{ {
throw Error("argument '%s' cannot be evaluated", what()); throw Error("argument '%s' cannot be evaluated", what());
} }
std::pair<Value *, Pos> toValue(EvalState & state);
/* Return a value only if this installable is a store path or a /* Return a value only if this installable is a store path or a
symlink to it. */ symlink to it. */
@@ -109,7 +110,7 @@ struct InstallableFlake : InstallableValue
std::vector<DerivationInfo> toDerivations() override; std::vector<DerivationInfo> toDerivations() override;
std::pair<Value *, Pos> toValue(EvalState & state) override; std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override;
std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
getCursors(EvalState & state) override; getCursors(EvalState & state) override;
+7 -10
View File
@@ -58,26 +58,23 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
Value * vNew = state.allocValue(); Value * vNew = state.allocValue();
state.autoCallFunction(autoArgs, *v, *vNew); state.autoCallFunction(autoArgs, *v, *vNew);
v = vNew; v = vNew;
state.forceValue(*v);
/* It should evaluate to either a set or an expression, /* It should evaluate to either a set or an expression,
according to what is specified in the attrPath. */ according to what is specified in the attrPath. */
if (!attrIndex) { if (!attrIndex) {
if (v->type() != nAttrs)
throw TypeError(
"the expression selected by the selection path '%1%' should be a set but is %2%",
attrPath,
showType(*v));
if (attr.empty()) if (attr.empty())
throw Error("empty attribute name in selection path '%1%'", attrPath); throw Error("empty attribute name in selection path '%1%'", attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); auto v2 = state.allocValue();
if (a == v->attrs->end()) auto gotField = state.lazyGetAttrField(*v, {state.symbols.create(attr)}, pos, *v2) != EvalState::LazyValueType::Missing;
if (!gotField)
throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
v = &*a->value; v = v2;
pos = *a->pos; /* Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); */
/* v = &*a->value; */
/* pos = *a->pos; */
} }
else { else {
+2
View File
@@ -2,6 +2,7 @@
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
#include "value-cache.hh"
#include <algorithm> #include <algorithm>
#include <optional> #include <optional>
@@ -36,6 +37,7 @@ class Bindings
public: public:
typedef uint32_t size_t; typedef uint32_t size_t;
Pos *pos; Pos *pos;
ValueCache eval_cache;
private: private:
size_t size_, capacity_; size_t size_, capacity_;
+21
View File
@@ -0,0 +1,21 @@
#include "context.hh"
namespace nix {
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
std::string encodeContext(std::string_view name, std::string_view path)
{
return "!" + std::string(name) + "!" + std::string(path);
}
}
+11
View File
@@ -0,0 +1,11 @@
#include "util.hh"
namespace nix {
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s);
std::string encodeContext(std::string_view name, std::string_view path);
}
+6
View File
@@ -46,6 +46,12 @@ void EvalState::forceValue(Value & v, const Pos & pos)
} }
else if (v.isApp()) else if (v.isApp())
callFunction(*v.app.left, *v.app.right, v, noPos); callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.isCachedThunk()) {
auto evalCache = v.getEvalCache();
v.mkThunk(v.cachedThunk.thunk->env, v.cachedThunk.thunk->expr);
forceValue(v, pos);
v.setEvalCache(evalCache);
}
else if (v.isBlackhole()) else if (v.isBlackhole())
throwEvalError(pos, "infinite recursion encountered"); throwEvalError(pos, "infinite recursion encountered");
} }
+368 -60
View File
@@ -1,4 +1,5 @@
#include "eval.hh" #include "eval.hh"
#include "value-cache.hh"
#include "hash.hh" #include "hash.hh"
#include "util.hh" #include "util.hh"
#include "store-api.hh" #include "store-api.hh"
@@ -8,6 +9,7 @@
#include "filetransfer.hh" #include "filetransfer.hh"
#include "json.hh" #include "json.hh"
#include "function-trace.hh" #include "function-trace.hh"
#include "value-cache.hh"
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
@@ -122,6 +124,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
str << "]"; str << "]";
break; break;
case tThunk: case tThunk:
case tCachedThunk:
case tApp: case tApp:
str << "<CODE>"; str << "<CODE>";
break; break;
@@ -193,7 +196,7 @@ string showType(const Value & v)
case tPrimOpApp: case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType(); case tExternal: return v.external->showType();
case tThunk: return "a thunk"; case tThunk: case tCachedThunk: return "a thunk";
case tApp: return "a function application"; case tApp: return "a function application";
case tBlackhole: return "a black hole"; case tBlackhole: return "a black hole";
default: default:
@@ -216,7 +219,7 @@ bool Value::isTrivial() const
return return
internalType != tApp internalType != tApp
&& internalType != tPrimOpApp && internalType != tPrimOpApp
&& (internalType != tThunk && ((internalType != tThunk && internalType != tCachedThunk)
|| (dynamic_cast<ExprAttrs *>(thunk.expr) || (dynamic_cast<ExprAttrs *>(thunk.expr)
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk.expr) || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -442,6 +445,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
EvalState::~EvalState() EvalState::~EvalState()
{ {
for (auto [_, cache] : evalCache) {
cache->commit();
}
} }
@@ -1107,68 +1113,235 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
v = *v2; v = *v2;
} }
unsigned long nrLookups = 0;
static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) std::pair<ValueCache::CacheResult, ValueCache> ValueCache::getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest)
{ {
std::ostringstream out; if (!rawCache)
bool first = true; return { {NoCacheKey}, ValueCache(nullptr) };
for (auto & i : attrPath) { auto resultingCursor = rawCache->findAlongAttrPath(selector);
if (!first) out << '.'; else first = false; if (!resultingCursor)
try { return { {CacheMiss}, ValueCache(nullptr) };
out << getName(i, state, env);
} catch (Error & e) { auto cachedValue = resultingCursor->getCachedValue();
assert(!i.symbol.set()); auto cacheResult = std::visit(
out << "\"${" << *i.expr << "}\""; overloaded{
} [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
} [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
return out.str(); [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
[&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
[&](tree_cache::missing_t x) {
return ValueCache::CacheResult{
.returnCode = CacheHit,
.lastQueriedSymbolIfMissing = x.attrName
};
},
[&](tree_cache::string_t s) {
PathSet context;
for (auto& [pathName, outputName] : s.second) {
// If the cached value depends on some non-existent
// path, we need to discard it and force the evaluation
// to bring back the context in the store
if (!state.store->isValidPath(
state.store->parseStorePath(pathName)))
return ValueCache::CacheResult{UnCacheable};
context.insert("!" + outputName + "!" + pathName);
}
mkString(dest, s.first, context);
return ValueCache::CacheResult{CacheHit};
},
[&](tree_cache::wrapped_basetype<bool> b) {
dest.mkBool(b.value);
return ValueCache::CacheResult{CacheHit};
},
[&](tree_cache::wrapped_basetype<int64_t> i) {
dest.mkInt(i.value);
return ValueCache::CacheResult{CacheHit};
},
[&](tree_cache::wrapped_basetype<double> d) {
dest.mkFloat(d.value);
return ValueCache::CacheResult{CacheHit};
},
},
cachedValue);
return { cacheResult, ValueCache(resultingCursor) };
} }
void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
{
switch (cacheResult.returnCode) {
case ValueCache::CacheHit:
nrCacheHits++;
break;
case ValueCache::CacheMiss:
nrCacheMisses++;
break;
case ValueCache::UnCacheable:
nrUncacheable++;
break;
case ValueCache::NoCacheKey:
nrUncached++;
break;
case ValueCache::Forward:
nrCacheHits++;
break;
};
}
unsigned long nrLookups = 0; EvalState::LazyValueType EvalState::lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
auto eval_cache = attrs.getEvalCache();
if (eval_cache.isEmpty()) {
forceValue(attrs, pos);
eval_cache = attrs.getEvalCache();
}
auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
updateCacheStats(cacheResult);
auto delayValue = [&](ValueCache & cursor) {
auto recordAsVar = new ExprCastedVar(&attrs);
auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
thunk->expr = accessExpr;
thunk->env = &baseEnv;
dest.mkCachedThunk(
thunk,
new ValueCache(cursor)
);
};
switch (cacheResult.returnCode) {
case ValueCache::CacheHit:
if (cacheResult.lastQueriedSymbolIfMissing)
return LazyValueType::Missing;
return LazyValueType::PlainValue;
case ValueCache::UnCacheable:
delayValue(resultingCursor);
return LazyValueType::DelayedUncacheable;
case ValueCache::Forward:
delayValue(resultingCursor);
return LazyValueType::DelayedAttr;
default:
if(getAttrField(attrs, selector, pos, dest))
return LazyValueType::PlainValue;
else
return LazyValueType::Missing;
}
}
bool EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
Pos * pos2 = 0;
auto eval_cache = attrs.getEvalCache();
if (eval_cache.isEmpty()) {
forceValue(attrs, pos);
eval_cache = attrs.getEvalCache();
}
auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
updateCacheStats(cacheResult);
switch (cacheResult.returnCode) {
case ValueCache::CacheHit:
if (cacheResult.lastQueriedSymbolIfMissing)
return false;
return true;
case ValueCache::CacheMiss:
resultingCursor = eval_cache;
break;
case ValueCache::Forward: // FIXME: Handle properly
case ValueCache::NoCacheKey:
case ValueCache::UnCacheable:
;
}
forceValue(attrs, pos);
Value * vAttrs = &attrs;
try {
for (auto & name : selector) {
nrLookups++;
Bindings::iterator j;
if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
return false;
}
vAttrs = j->value;
pos2 = j->pos;
try {
forceValue(*vAttrs, pos2 != NULL ? *pos2 : pos );
} catch (EvalError & e) {
resultingCursor.addFailedChild(name, e);
throw;
}
if (cacheResult.returnCode == ValueCache::CacheMiss) {
resultingCursor = resultingCursor.addChild(name, *vAttrs);
vAttrs->setEvalCache(resultingCursor);
}
if (countCalls && pos2) attrSelects[*pos2]++;
}
} catch (Error & e) {
if (pos2 && pos2->file != sDerivationNix) {
vector<string> strSelector;
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", concatStringsSep(".", selector));
}
throw;
}
if (cacheResult.returnCode == ValueCache::Forward)
vAttrs->setEvalCache(resultingCursor);
dest = *vAttrs;
return true;
}
void EvalState::getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
{
if (!getAttrField(attrs, selector, pos, dest))
throw Error("Missing attribute path '%s'", "ImTooLazyToImplementThisRightNow");
}
std::vector<Symbol> EvalState::getFields(Value & attrs, const Pos & pos)
{
auto eval_cache = attrs.getEvalCache();
if (eval_cache.isEmpty()) {
forceValue(attrs, pos);
eval_cache = attrs.getEvalCache();
}
if (auto attrNames = eval_cache.listChildren(symbols)) {
return *attrNames;
}
forceAttrs(attrs);
std::vector<Symbol> res;
for (auto & attr : *attrs.attrs)
res.push_back(attr.name);
return res;
}
void ExprSelect::eval(EvalState & state, Env & env, Value & v) void ExprSelect::eval(EvalState & state, Env & env, Value & v)
{ {
Value vTmp; Value vTmp;
Pos * pos2 = 0;
Value * vAttrs = &vTmp;
e->eval(state, env, vTmp); e->eval(state, env, vTmp);
try { std::vector<Symbol> selector;
for (auto & i : attrPath)
selector.push_back(getName(i, state, env));
for (auto & i : attrPath) { bool gotField = state.getAttrField(vTmp, selector, pos, v);
nrLookups++; if (!gotField) {
Bindings::iterator j; if (def) {
Symbol name = getName(i, state, env); def->eval(state, env, v);
if (def) { return;
state.forceValue(*vAttrs, pos); } else {
if (vAttrs->type() != nAttrs || throwEvalError(pos, "Missing field");
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
def->eval(state, env, v);
return;
}
} else {
state.forceAttrs(*vAttrs, pos);
if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
throwEvalError(pos, "attribute '%1%' missing", name);
}
vAttrs = j->value;
pos2 = j->pos;
if (state.countCalls && pos2) state.attrSelects[*pos2]++;
} }
state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
} catch (Error & e) {
if (pos2 && pos2->file != state.sDerivationNix)
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath));
throw;
} }
v = *vAttrs;
} }
@@ -1351,9 +1524,38 @@ void EvalState::incrFunctionCall(ExprLambda * fun)
functionCalls[fun]++; functionCalls[fun]++;
} }
std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
{
if (!rawCache)
return std::nullopt;
return rawCache->getCachedValue();
}
std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
{
auto ret = std::vector<Symbol>();
if (rawCache) {
auto cachedValue = rawCache->getCachedValue();
if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
for (auto & fieldStr : rawCache->getChildren())
ret.push_back(symbols.create(fieldStr));
}
return ret;
}
return std::nullopt;
}
void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
{ {
if (auto evalCache = fun.getEvalCache(); !evalCache.isEmpty()) {
if (auto cacheValue = evalCache.getRawValue()) {
if (std::holds_alternative<tree_cache::attributeSet_t>(*cacheValue)) {
res = fun;
return;
}
}
}
forceValue(fun); forceValue(fun);
if (fun.type() == nAttrs) { if (fun.type() == nAttrs) {
@@ -1699,18 +1901,6 @@ string EvalState::forceString(Value & v, const Pos & pos)
} }
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
} else
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
}
void copyContext(const Value & v, PathSet & context) void copyContext(const Value & v, PathSet & context)
{ {
if (v.string.context) if (v.string.context)
@@ -1729,6 +1919,102 @@ std::vector<std::pair<Path, std::string>> Value::getContext()
return res; return res;
} }
ValueCache & Value::getEvalCache()
{
if (internalType == tAttrs) {
return attrs->eval_cache;
} else if (internalType == tCachedThunk) {
return *cachedThunk.cache;
} else {
return ValueCache::empty;
}
}
ValueCache ValueCache::empty = ValueCache(nullptr);
tree_cache::AttrValue cachedValueFor(Value& v)
{
tree_cache::AttrValue valueToCache;
switch (v.type()) {
case nThunk:
valueToCache = tree_cache::thunk_t{};
break;
case nNull:
case nList:
case nFunction:
case nExternal:
valueToCache = tree_cache::unknown_t{};
break;
case nBool:
valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
break;
case nString:
valueToCache = tree_cache::string_t{
v.string.s,
v.getContext()
};
break;
case nPath:
valueToCache = tree_cache::string_t{
v.path,
{}
};
break;
case nAttrs:
valueToCache = tree_cache::attributeSet_t{};
break;
case nInt:
valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
break;
case nFloat:
valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
break;
};
return valueToCache;
}
ValueCache ValueCache::addChild(const Symbol& name, Value& value)
{
if (!rawCache)
return ValueCache::empty;
auto cachedValue = cachedValueFor(value);
auto ret = ValueCache(rawCache->addChild(name, cachedValue));
if (value.type() == nAttrs)
ret.addAttrSetChilds(*value.attrs);
return ret;
}
ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
{
if (!rawCache) return ValueCache::empty;
return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
}
void ValueCache::addAttrSetChilds(Bindings & children)
{
if (!rawCache) return;
for (auto & attr : children) {
// We could in theory directly store the value, but that would cause
// an infinite recursion in case of a cyclic attrset.
// So in a first time, just store a thunk
if (attr.value->type() == nAttrs)
rawCache->addChild(attr.name, tree_cache::thunk_t{});
else
addChild(attr.name, *attr.value);
}
}
void Value::setEvalCache(ValueCache & newCache)
{
if (internalType == tAttrs) {
attrs->eval_cache = newCache;
} else if (internalType == tCachedThunk) {
cachedThunk.cache = &newCache;
}
}
string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
{ {
@@ -2011,6 +2297,13 @@ void EvalState::printStats()
topObj.attr("nrLookups", nrLookups); topObj.attr("nrLookups", nrLookups);
topObj.attr("nrPrimOpCalls", nrPrimOpCalls); topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
topObj.attr("nrFunctionCalls", nrFunctionCalls); topObj.attr("nrFunctionCalls", nrFunctionCalls);
{
auto cache = topObj.object("evalCache");
cache.attr("nrCacheMisses", nrCacheMisses);
cache.attr("nrCacheHits", nrCacheHits);
cache.attr("nrUncached", nrUncached);
cache.attr("nrUncacheable", nrUncacheable);
}
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
{ {
auto gc = topObj.object("gc"); auto gc = topObj.object("gc");
@@ -2111,6 +2404,21 @@ Strings EvalSettings::getDefaultNixPath()
return res; return res;
} }
std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
{
if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
return iter->second;
if (!(evalSettings.useEvalCache && evalSettings.pureEval))
return nullptr;
auto thisCache = tree_cache::Cache::tryCreate(
cacheKey,
symbols
);
evalCache.insert({cacheKey, thisCache});
return thisCache;
}
EvalSettings evalSettings; EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings); static GlobalConfig::Register rEvalSettings(&evalSettings);
+31 -4
View File
@@ -1,6 +1,7 @@
#pragma once #pragma once
#include "attr-set.hh" #include "attr-set.hh"
#include "context.hh"
#include "value.hh" #include "value.hh"
#include "nixexpr.hh" #include "nixexpr.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
@@ -115,6 +116,7 @@ private:
typedef std::map<Path, Value> FileEvalCache; typedef std::map<Path, Value> FileEvalCache;
#endif #endif
FileEvalCache fileEvalCache; FileEvalCache fileEvalCache;
std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
SearchPath searchPath; SearchPath searchPath;
@@ -131,6 +133,8 @@ public:
EvalState(const Strings & _searchPath, ref<Store> store); EvalState(const Strings & _searchPath, ref<Store> store);
~EvalState(); ~EvalState();
std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
void addToSearchPath(const string & s); void addToSearchPath(const string & s);
SearchPath getSearchPath() { return searchPath; } SearchPath getSearchPath() { return searchPath; }
@@ -310,6 +314,8 @@ public:
void realiseContext(const PathSet & context); void realiseContext(const PathSet & context);
void updateCacheStats(ValueCache::CacheResult);
private: private:
unsigned long nrEnvs = 0; unsigned long nrEnvs = 0;
@@ -323,6 +329,10 @@ private:
unsigned long nrListConcats = 0; unsigned long nrListConcats = 0;
unsigned long nrPrimOpCalls = 0; unsigned long nrPrimOpCalls = 0;
unsigned long nrFunctionCalls = 0; unsigned long nrFunctionCalls = 0;
unsigned long nrCacheHits = 0;
unsigned long nrCacheMisses = 0;
unsigned long nrUncacheable = 0;
unsigned long nrUncached = 0;
bool countCalls; bool countCalls;
@@ -342,6 +352,27 @@ private:
friend struct ExprSelect; friend struct ExprSelect;
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
public:
bool getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
enum struct LazyValueType {
PlainValue,
DelayedUncacheable,
DelayedAttr,
Missing,
};
// Similar to `getAttrField`, but if the cache says that the result is an
// attribute set, just return a thunk to it rather than forcing it.
LazyValueType lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
// Similar to `getAttrField`, but throws an `Error` if the field can’t be
// found
void getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
std::vector<Symbol> getFields(Value & attrs, const Pos & pos);
}; };
@@ -349,10 +380,6 @@ private:
string showType(ValueType type); string showType(ValueType type);
string showType(const Value & v); string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
std::pair<string, string> decodeContext(std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */ /* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path); Path resolveExprPath(Path path);
+32 -18
View File
@@ -596,30 +596,44 @@ void callFlake(EvalState & state,
const LockedFlake & lockedFlake, const LockedFlake & lockedFlake,
Value & vRes) Value & vRes)
{ {
auto vLocks = state.allocValue(); auto lockExpr = new ExprString(
state.symbols.create(lockedFlake.lockFile.to_string())
);
auto subdirExpr = new ExprString(
state.symbols.create(lockedFlake.flake.lockedRef.subdir)
);
auto vRootSrc = state.allocValue(); auto vRootSrc = state.allocValue();
auto vRootSubdir = state.allocValue();
auto vTmp1 = state.allocValue();
auto vTmp2 = state.allocValue();
mkString(*vLocks, lockedFlake.lockFile.to_string());
emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); static Expr * callFlakeExpr = nullptr;
if (!callFlakeExpr) {
static RootValue vCallFlake = nullptr; callFlakeExpr = state.parseExprFromString(
if (!vCallFlake) {
vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh" #include "call-flake.nix.gen.hh"
, "/"), **vCallFlake); , "/");
} }
state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos); auto resExpr = new ExprApp(
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); new ExprApp(
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); new ExprApp(
callFlakeExpr,
lockExpr
),
new ExprCastedVar(vRootSrc)
),
subdirExpr
);
auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
thunk->expr = resExpr;
thunk->env = &state.baseEnv;
auto fingerprint = lockedFlake.getFingerprint();
auto treeCache = state.openTreeCache(fingerprint);
auto cacheRoot = treeCache ? treeCache->getRoot() : nullptr;
auto evalCache = new ValueCache(cacheRoot);
vRes.mkCachedThunk(
thunk,
evalCache
);
} }
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
+12
View File
@@ -440,6 +440,18 @@ string ExprLambda::showNamePos() const
return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
} }
void ExprCastedVar::show(std::ostream & str) const {
std::set<const Value*> active;
printValue(str, active, *v);
}
void ExprCastedVar::bindVars(const StaticEnv & env) {}
void ExprCastedVar::eval(EvalState & state, Env & env, Value & v) {
v = std::move(*this->v);
}
Value * ExprCastedVar::maybeThunk(EvalState & state, Env & env) {
return v;
}
/* Symbol table. */ /* Symbol table. */
+14 -1
View File
@@ -165,7 +165,12 @@ struct ExprSelect : Expr
Expr * e, * def; Expr * e, * def;
AttrPath attrPath; AttrPath attrPath;
ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; ExprSelect(const Pos & pos, Expr * e, const Symbol & name)
: ExprSelect(pos, e, std::vector{name}) {};
ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & symbolicAttrPath) : pos(pos), e(e), def(0) {
for (auto & name : symbolicAttrPath)
attrPath.push_back(AttrName(name));
};
COMMON_METHODS COMMON_METHODS
}; };
@@ -334,6 +339,14 @@ struct ExprPos : Expr
COMMON_METHODS COMMON_METHODS
}; };
struct ExprCastedVar : Expr
{
Value * v;
ExprCastedVar(Value * v) : v(v) {};
Value * maybeThunk(EvalState & state, Env & env);
COMMON_METHODS
};
/* Static environments are used to map variable names onto (level, /* Static environments are used to map variable names onto (level,
displacement) pairs used to obtain the value of the variable at displacement) pairs used to obtain the value of the variable at
+392
View File
@@ -0,0 +1,392 @@
#include "tree-cache.hh"
#include "sqlite.hh"
#include "store-api.hh"
#include "context.hh"
namespace nix::tree_cache {
static const char * schema = R"sql(
create table if not exists Attributes (
id integer primary key autoincrement not null,
parent integer not null,
name text,
type integer not null,
value text,
context text,
unique (parent, name)
);
create index if not exists IndexByParent on Attributes(parent, name);
)sql";
struct AttrDb
{
std::atomic_bool failed{false};
struct State
{
SQLite db;
SQLiteStmt insertAttribute;
SQLiteStmt updateAttribute;
SQLiteStmt insertAttributeWithContext;
SQLiteStmt queryAttribute;
SQLiteStmt queryAttributes;
std::unique_ptr<SQLiteTxn> txn;
};
std::unique_ptr<Sync<State>> _state;
AttrDb(const Hash & fingerprint)
: _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->insertAttribute.create(state->db,
"insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
state->updateAttribute.create(state->db,
"update Attributes set type = ?, value = ?, context = ? where id = ?");
state->insertAttributeWithContext.create(state->db,
"insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
state->queryAttribute.create(state->db,
"select id, type, value, context from Attributes where parent = ? and name = ?");
state->queryAttributes.create(state->db,
"select name, type from Attributes where parent = ?");
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
~AttrDb()
{
try {
auto state(_state->lock());
if (!failed)
state->txn->commit();
state->txn.reset();
} catch (...) {
ignoreException();
}
}
template<typename F>
AttrId doSQLite(F && fun)
{
if (failed) return 0;
try {
return fun();
} catch (SQLiteError &) {
ignoreException();
failed = true;
return 0;
}
}
/**
* Store a leaf of the tree in the db
*/
AttrId addEntry(
const AttrKey & key,
const AttrValue & value)
{
return doSQLite([&]()
{
auto state(_state->lock());
auto rawValue = RawValue::fromVariant(value);
state->insertAttributeWithContext.use()
(key.first)
(key.second)
(rawValue.type)
(rawValue.value.value_or(""), rawValue.value.has_value())
(rawValue.serializeContext())
.exec();
AttrId rowId = state->db.getLastInsertedRowId();
assert(rowId);
return rowId;
});
}
std::optional<AttrId> getId(const AttrKey& key)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return std::nullopt;
return (AttrType) queryAttribute.getInt(0);
}
AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
{
debug("cache: miss for the attribute %s", key.second);
if (auto existingId = getId(key)) {
setValue(*existingId, value);
return *existingId;
}
return addEntry(key, value);
}
void setValue(const AttrId & id, const AttrValue & value)
{
auto state(_state->lock());
auto rawValue = RawValue::fromVariant(value);
state->updateAttribute.use()
(rawValue.type)
(rawValue.value.value_or(""), rawValue.value.has_value())
(rawValue.serializeContext())
(id)
.exec();
}
std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
{
auto state(_state->lock());
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
if (!queryAttribute.next()) return {};
auto rowId = (AttrType) queryAttribute.getInt(0);
auto type = (AttrType) queryAttribute.getInt(1);
switch (type) {
case AttrType::Attrs: {
return {{rowId, attributeSet_t()}};
}
case AttrType::String: {
std::vector<std::pair<Path, std::string>> context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.push_back(decodeContext(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
case AttrType::Int:
return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
case AttrType::Double:
return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
case AttrType::Unknown:
return {{rowId, unknown_t{}}};
case AttrType::Thunk:
return {{rowId, thunk_t{}}};
case AttrType::Missing:
return {{rowId, missing_t{key.second}}};
case AttrType::Failed:
return {{rowId, failed_t{queryAttribute.getStr(2)}}};
default:
throw Error("unexpected type in evaluation cache");
}
}
std::vector<std::string> getChildren(AttrId parentId)
{
std::vector<std::string> res;
auto state(_state->lock());
auto queryAttributes(state->queryAttributes.use()(parentId));
while (queryAttributes.next()) {
if (queryAttributes.getInt(1) != AttrType::Missing)
res.push_back(queryAttributes.getStr(0));
}
return res;
}
};
Cache::Cache(const Hash & useCache,
SymbolTable & symbols)
: db(std::make_shared<AttrDb>(useCache))
, symbols(symbols)
, rootSymbol(symbols.create("%root%"))
{
}
std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
{
try {
return std::make_shared<Cache>(useCache, symbols);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
}
}
void Cache::commit()
{
if (db) {
debug("Saving the cache");
auto state(db->_state->lock());
if (state->txn->active) {
state->txn->commit();
state->txn.reset();
state->txn = std::make_unique<SQLiteTxn>(state->db);
}
}
}
Cursor::Ref Cache::getRoot()
{
return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
}
Cursor::Cursor(
ref<Cache> root,
const Parent & parent,
const AttrValue& value
)
: root(root)
, parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
, label(parent ? parent->second : root->rootSymbol)
, cachedValue({root->db->setOrUpdate(getKey(), value), value})
{
}
Cursor::Cursor(
ref<Cache> root,
const Parent & parent,
const AttrId & id,
const AttrValue & value
)
: root(root)
, parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
, label(parent ? parent->second : root->rootSymbol)
, cachedValue({id, value})
{
}
AttrKey Cursor::getKey()
{
if (!parentId)
return {0, root->rootSymbol};
return {*parentId, label};
}
AttrValue Cursor::getCachedValue()
{
return cachedValue.second;
}
void Cursor::setValue(const AttrValue & v)
{
root->db->setValue(cachedValue.first, v);
cachedValue.second = v;
}
Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
{
Parent parent = {{*this, attrPath}};
auto childCursor = new Cursor(
root,
parent,
v
);
return childCursor;
}
std::vector<std::string> Cursor::getChildren()
{
return root->db->getChildren(cachedValue.first);
}
std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
{
auto cursorAtPath = findAlongAttrPath(attrPath);
if (cursorAtPath)
return cursorAtPath->getChildren();
return std::nullopt;
}
Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
{
auto rawAttr = root->db->getValue({cachedValue.first, name});
if (rawAttr) {
Parent parent = {{*this, name}};
debug("cache: hit for the attribute %s", cachedValue.first);
return new Cursor (
root, parent, rawAttr->first,
rawAttr->second);
}
if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
// If the parent is an attribute set but we're not present in the db,
// then we're not a member of this attribute set. So mark as missing
return addChild(name, missing_t{name});
}
return nullptr;
}
Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto currentCursor = this;
for (auto & currentAccessor : attrPath) {
currentCursor = currentCursor->maybeGetAttr(currentAccessor);
if (!currentCursor)
break;
if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
break;
if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
break;
}
return currentCursor;
}
const RawValue RawValue::fromVariant(const AttrValue & value)
{
RawValue res;
std::visit(overloaded{
[&](attributeSet_t x) { res.type = AttrType::Attrs; },
[&](string_t x) {
res.type = AttrType::String;
res.value = x.first;
res.context = x.second;
},
[&](wrapped_basetype<bool> x) {
res.type = AttrType::Bool;
res.value = x.value ? "1" : "0";
},
[&](wrapped_basetype<int64_t> x) {
res.type = AttrType::Int;
res.value = std::to_string(x.value);
},
[&](wrapped_basetype<double> x) {
res.type = AttrType::Double;
res.value = std::to_string(x.value);
},
[&](unknown_t x) { res.type = AttrType::Unknown; },
[&](missing_t x) { res.type = AttrType::Missing; },
[&](thunk_t x) { res.type = AttrType::Thunk; },
[&](failed_t x) {
res.type = AttrType::Failed;
res.value = x.error;
}
}, value);
return res;
}
std::string RawValue::serializeContext() const
{
std::string res;
for (auto & elt : context) {
res.append(encodeContext(elt.second, elt.first));
res.push_back(' ');
}
if (!res.empty())
res.pop_back(); // Remove the trailing space
return res;
}
}
+156
View File
@@ -0,0 +1,156 @@
/**
* caching for a tree-like data structure (like Nix values)
*
* The cache is an sqlite db whose rows are the nodes of the tree, with a
* pointer to their parent (except for the root of course)
*/
#pragma once
#include "sync.hh"
#include "hash.hh"
#include "symbol-table.hh"
#include <functional>
#include <variant>
namespace nix::tree_cache {
struct AttrDb;
class Cursor;
class Cache : public std::enable_shared_from_this<Cache>
{
private:
friend class Cursor;
/**
* The database holding the cache
*/
std::shared_ptr<AttrDb> db;
SymbolTable & symbols;
/**
* Distinguished symbol indicating the root of the tree
*/
const Symbol rootSymbol;
public:
Cache(
const Hash & useCache,
SymbolTable & symbols
);
static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
Cursor * getRoot();
/**
* Flush the cache to disk
*/
void commit();
};
enum AttrType {
Unknown = 0,
Attrs = 1,
String = 2,
Bool = 3,
Int = 4,
Double = 5,
Thunk = 6,
Missing = 7, // Missing fields of attribute sets
Failed = 8,
};
struct attributeSet_t {};
struct unknown_t {};
struct thunk_t {};
struct failed_t { string error; };
struct missing_t { Symbol attrName; };
// Putting several different primitive types in an `std::variant` partially
// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
// from one to another which breaks the exhaustiveness check.
// So we wrap them in a trivial class just to force the disambiguation
template<typename T>
struct wrapped_basetype{ T value; };
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
typedef std::variant<
attributeSet_t,
string_t,
unknown_t,
thunk_t,
missing_t,
failed_t,
wrapped_basetype<bool>,
wrapped_basetype<int64_t>,
wrapped_basetype<double>
> AttrValue;
struct RawValue {
AttrType type;
std::optional<std::string> value;
std::vector<std::pair<Path, std::string>> context;
std::string serializeContext() const;
static const RawValue fromVariant(const AttrValue&);
AttrValue toVariant() const;
};
/**
* View inside the cache.
*
* A `Cursor` represents a node in the cached tree (be it a leaf or not)
*/
class Cursor : public std::enable_shared_from_this<Cursor>
{
/**
* The overall cache of which this cursor is a view
*/
ref<Cache> root;
typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
std::optional<AttrId> parentId;
Symbol label;
std::pair<AttrId, AttrValue> cachedValue;
/**
* Get the identifier for this node in the database
*/
AttrKey getKey();
public:
using Ref = Cursor*;
// Create a new cache entry
Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
// Build a cursor from an existing cache entry
Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
AttrValue getCachedValue();
void setValue(const AttrValue & v);
Ref addChild(const Symbol & attrPath, const AttrValue & v);
Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
Ref maybeGetAttr(const Symbol & attrPath);
std::vector<std::string> getChildren();
std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
};
}
+51
View File
@@ -0,0 +1,51 @@
#pragma once
#include "tree-cache.hh"
namespace nix {
struct Value;
class EvalState;
class Bindings;
class ValueCache {
tree_cache::Cursor::Ref rawCache;
public:
ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
static ValueCache empty;
bool isEmpty () { return rawCache == nullptr; }
enum ReturnCode {
// The cache result was an attribute set, so we forward it later in the
// chain
Forward,
CacheMiss,
CacheHit,
UnCacheable,
NoCacheKey,
};
struct CacheResult {
ReturnCode returnCode;
// In case the query returns a `missing_t`, the symbol that's missing
std::optional<Symbol> lastQueriedSymbolIfMissing;
};
std::pair<CacheResult, ValueCache> getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest);
ValueCache addChild(const Symbol & attrName, Value & value);
ValueCache addFailedChild(const Symbol & attrName, const Error & error);
ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
void addAttrSetChilds(Bindings & children);
void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
std::optional<tree_cache::AttrValue> getRawValue();
ValueCache() : rawCache(nullptr) {}
};
}
+24 -4
View File
@@ -23,6 +23,7 @@ typedef enum {
tApp, tApp,
tLambda, tLambda,
tBlackhole, tBlackhole,
tCachedThunk,
tPrimOp, tPrimOp,
tPrimOpApp, tPrimOpApp,
tExternal, tExternal,
@@ -56,6 +57,7 @@ struct Pos;
class EvalState; class EvalState;
class XMLWriter; class XMLWriter;
class JSONPlaceholder; class JSONPlaceholder;
class ValueCache;
typedef int64_t NixInt; typedef int64_t NixInt;
@@ -103,6 +105,10 @@ class ExternalValueBase
std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
struct Thunk {
Env * env;
Expr * expr;
};
struct Value struct Value
{ {
@@ -122,6 +128,7 @@ public:
inline bool isThunk() const { return internalType == tThunk; }; inline bool isThunk() const { return internalType == tThunk; };
inline bool isApp() const { return internalType == tApp; }; inline bool isApp() const { return internalType == tApp; };
inline bool isBlackhole() const { return internalType == tBlackhole; }; inline bool isBlackhole() const { return internalType == tBlackhole; };
inline bool isCachedThunk() const { return internalType == tCachedThunk; };
// type() == nFunction // type() == nFunction
inline bool isLambda() const { return internalType == tLambda; }; inline bool isLambda() const { return internalType == tLambda; };
@@ -165,10 +172,13 @@ public:
Value * * elems; Value * * elems;
} bigList; } bigList;
Value * smallList[2]; Value * smallList[2];
Thunk thunk;
struct { struct {
Env * env; Thunk * thunk;
Expr * expr; // This is a pointer only to prevent a recursive import as
} thunk; // `EvalCache` is already a pointer so would fit very nicely here.
ValueCache * cache;
} cachedThunk;
struct { struct {
Value * left, * right; Value * left, * right;
} app; } app;
@@ -199,7 +209,7 @@ public:
case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
case tExternal: return nExternal; case tExternal: return nExternal;
case tFloat: return nFloat; case tFloat: return nFloat;
case tThunk: case tApp: case tBlackhole: return nThunk; case tThunk: case tApp: case tBlackhole: case tCachedThunk: return nThunk;
} }
abort(); abort();
} }
@@ -272,6 +282,13 @@ public:
thunk.expr = ex; thunk.expr = ex;
} }
inline void mkCachedThunk(Thunk * t, ValueCache * cache)
{
internalType = tCachedThunk;
cachedThunk.thunk = t;
cachedThunk.cache = cache;
}
inline void mkApp(Value * l, Value * r) inline void mkApp(Value * l, Value * r)
{ {
internalType = tApp; internalType = tApp;
@@ -349,6 +366,9 @@ public:
bool isTrivial() const; bool isTrivial() const;
std::vector<std::pair<Path, std::string>> getContext(); std::vector<std::pair<Path, std::string>> getContext();
ValueCache & getEvalCache();
void setEvalCache(ValueCache &);
}; };
+51 -77
View File
@@ -739,63 +739,6 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
{ {
} }
void runPostBuildHook(
Store & store,
Logger & logger,
const StorePath & drvPath,
StorePathSet outputPaths
)
{
auto hook = settings.postBuildHook;
if (hook == "")
return;
Activity act(logger, lvlInfo, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
RunOptions opts(settings.postBuildHook, {});
opts.environment = hookEnvironment;
struct LogSink : Sink {
Activity & act;
std::string currentLine;
LogSink(Activity & act) : act(act) { }
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
currentLine += c;
}
}
}
void flushLine() {
act.result(resPostBuildLogLine, currentLine);
currentLine.clear();
}
~LogSink() {
if (currentLine != "") {
currentLine += '\n';
flushLine();
}
}
};
LogSink sink(act);
opts.standardOut = &sink;
opts.mergeStderrToStdout = true;
runProgram2(opts);
}
void DerivationGoal::buildDone() void DerivationGoal::buildDone()
{ {
@@ -861,15 +804,57 @@ void DerivationGoal::buildDone()
being valid. */ being valid. */
registerOutputs(); registerOutputs();
StorePathSet outputPaths; if (settings.postBuildHook != "") {
for (auto & [_, path] : finalOutputs) Activity act(*logger, lvlInfo, actPostBuildHook,
outputPaths.insert(path); fmt("running post-build-hook '%s'", settings.postBuildHook),
runPostBuildHook( Logger::Fields{worker.store.printStorePath(drvPath)});
worker.store, PushActivity pact(act.id);
*logger, StorePathSet outputPaths;
drvPath, for (auto i : drv->outputs) {
outputPaths outputPaths.insert(finalOutputs.at(i.first));
); }
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
RunOptions opts(settings.postBuildHook, {});
opts.environment = hookEnvironment;
struct LogSink : Sink {
Activity & act;
std::string currentLine;
LogSink(Activity & act) : act(act) { }
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
currentLine += c;
}
}
}
void flushLine() {
act.result(resPostBuildLogLine, currentLine);
currentLine.clear();
}
~LogSink() {
if (currentLine != "") {
currentLine += '\n';
flushLine();
}
}
};
LogSink sink(act);
opts.standardOut = &sink;
opts.mergeStderrToStdout = true;
runProgram2(opts);
}
if (buildMode == bmCheck) { if (buildMode == bmCheck) {
cleanupPostOutputsRegisteredModeCheck(); cleanupPostOutputsRegisteredModeCheck();
@@ -925,8 +910,6 @@ void DerivationGoal::resolvedFinished() {
auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs” // `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs; auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty()) if (realWantedOutputs.empty())
@@ -944,10 +927,8 @@ void DerivationGoal::resolvedFinished() {
auto newRealisation = *realisation; auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
newRealisation.signatures.clear(); newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation); signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
} else { } else {
// If we don't have a realisation, then it must mean that something // If we don't have a realisation, then it must mean that something
// failed when building the resolved drv // failed when building the resolved drv
@@ -955,13 +936,6 @@ void DerivationGoal::resolvedFinished() {
} }
} }
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
// This is potentially a bit fishy in terms of error reporting. Not sure // This is potentially a bit fishy in terms of error reporting. Not sure
// how to do it in a cleaner way // how to do it in a cleaner way
amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
@@ -17,13 +17,6 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
void DrvOutputSubstitutionGoal::init() void DrvOutputSubstitutionGoal::init()
{ {
trace("init"); trace("init");
/* If the derivation already exists, we’re done */
if (worker.store.queryRealisation(id)) {
amDone(ecSuccess);
return;
}
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext(); tryNext();
} }
@@ -60,26 +53,6 @@ void DrvOutputSubstitutionGoal::tryNext()
return; return;
} }
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
localOutputInfo && localOutputInfo->outPath != depPath) {
warn(
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
"Local: %s\n"
"Remote: %s",
sub->getUri(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath)
);
tryNext();
return;
}
addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
}
}
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
if (waitees.empty()) outPathValid(); if (waitees.empty()) outPathValid();
+5 -24
View File
@@ -292,7 +292,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
auto & localStore = getLocalStore(); auto & localStore = getLocalStore();
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st; struct statvfs st;
if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
(uint64_t) st.f_bavail * st.f_bsize < required) (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true; diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 && if (statvfs(tmpDir.c_str(), &st) == 0 &&
@@ -417,7 +417,7 @@ void LocalDerivationGoal::startBuilder()
} }
auto & localStore = getLocalStore(); auto & localStore = getLocalStore();
if (localStore.storeDir != localStore.realStoreDir.get()) { if (localStore.storeDir != localStore.realStoreDir) {
#if __linux__ #if __linux__
useChroot = true; useChroot = true;
#else #else
@@ -726,7 +726,7 @@ void LocalDerivationGoal::startBuilder()
Path logFile = openLogFile(); Path logFile = openLogFile();
/* Create a pipe to get the output of the builder. */ /* Create a pipe to get the output of the builder. */
builderOut.create(); //builderOut.create();
builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
if (!builderOut.readSide) if (!builderOut.readSide)
@@ -1333,18 +1333,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
// XXX: This should probably be allowed if the realisation corresponds to // XXX: This should probably be allowed if the realisation corresponds to
// an allowed derivation // an allowed derivation
{ { throw Error("queryRealisation"); }
if (!goal.isAllowed(id))
throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
return next->queryRealisation(id);
}
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
{ {
if (buildMode != bmNormal) throw Error("unsupported build mode"); if (buildMode != bmNormal) throw Error("unsupported build mode");
StorePathSet newPaths; StorePathSet newPaths;
std::set<Realisation> newRealisations;
for (auto & req : paths) { for (auto & req : paths) {
if (!goal.isAllowed(req)) if (!goal.isAllowed(req))
@@ -1357,28 +1352,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
auto p = std::get_if<DerivedPath::Built>(&path); auto p = std::get_if<DerivedPath::Built>(&path);
if (!p) continue; if (!p) continue;
auto & bfd = *p; auto & bfd = *p;
auto drv = readDerivation(bfd.drvPath);
auto drvHashes = staticOutputHashes(*this, drv);
auto outputs = next->queryDerivationOutputMap(bfd.drvPath); auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
for (auto & [outputName, outputPath] : outputs) for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs)) { if (wantOutput(outputName, bfd.outputs))
newPaths.insert(outputPath); newPaths.insert(outputPath);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto thisRealisation = next->queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}
);
assert(thisRealisation);
newRealisations.insert(*thisRealisation);
}
}
} }
StorePathSet closure; StorePathSet closure;
next->computeFSClosure(newPaths, closure); next->computeFSClosure(newPaths, closure);
for (auto & path : closure) for (auto & path : closure)
goal.addDependency(path); goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id);
} }
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -2481,7 +2464,6 @@ void LocalDerivationGoal::registerOutputs()
floating CA derivations and hash-mismatching fixed-output floating CA derivations and hash-mismatching fixed-output
derivations. */ derivations. */
PathLocks dynamicOutputLock; PathLocks dynamicOutputLock;
dynamicOutputLock.setDeletion(true);
auto optFixedPath = output.path(worker.store, drv->name, outputName); auto optFixedPath = output.path(worker.store, drv->name, outputName);
if (!optFixedPath || if (!optFixedPath ||
worker.store.printStorePath(*optFixedPath) != finalDestPath) worker.store.printStorePath(*optFixedPath) != finalDestPath)
@@ -2505,7 +2487,6 @@ void LocalDerivationGoal::registerOutputs()
assert(newInfo.ca); assert(newInfo.ca);
} else { } else {
auto destPath = worker.store.toRealPath(finalDestPath); auto destPath = worker.store.toRealPath(finalDestPath);
deletePath(destPath);
movePath(actualPath, destPath); movePath(actualPath, destPath);
actualPath = destPath; actualPath = destPath;
} }
@@ -108,9 +108,6 @@ struct LocalDerivationGoal : public DerivationGoal
/* Paths that were added via recursive Nix calls. */ /* Paths that were added via recursive Nix calls. */
StorePathSet addedPaths; StorePathSet addedPaths;
/* Realisations that were added via recursive Nix calls. */
std::set<DrvOutput> addedDrvOutputs;
/* Recursive Nix calls are only allowed to build or realize paths /* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@@ -119,11 +116,6 @@ struct LocalDerivationGoal : public DerivationGoal
{ {
return inputPaths.count(path) || addedPaths.count(path); return inputPaths.count(path) || addedPaths.count(path);
} }
bool isAllowed(const DrvOutput & id)
{
return addedDrvOutputs.count(id);
}
bool isAllowed(const DerivedPath & req); bool isAllowed(const DerivedPath & req);
friend struct RestrictedStore; friend struct RestrictedStore;
+1 -10
View File
@@ -3,19 +3,10 @@
-- is enabled -- is enabled
create table if not exists Realisations ( create table if not exists Realisations (
id integer primary key autoincrement not null,
drvPath text not null, drvPath text not null,
outputName text not null, -- symbolic output id, usually "out" outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null, outputPath integer not null,
signatures text, -- space-separated list signatures text, -- space-separated list
primary key (drvPath, outputName),
foreign key (outputPath) references ValidPaths(id) on delete cascade foreign key (outputPath) references ValidPaths(id) on delete cascade
); );
create index if not exists IndexRealisations on Realisations(drvPath, outputName);
create table if not exists RealisationsRefs (
referrer integer not null,
realisationReference integer,
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);
+7 -18
View File
@@ -885,15 +885,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopRegisterDrvOutput: { case wopRegisterDrvOutput: {
logger->startWork(); logger->startWork();
if (GET_PROTOCOL_MINOR(clientVersion) < 31) { auto outputId = DrvOutput::parse(readString(from));
auto outputId = DrvOutput::parse(readString(from)); auto outputPath = StorePath(readString(from));
auto outputPath = StorePath(readString(from)); store->registerDrvOutput(Realisation{
store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath});
.id = outputId, .outPath = outputPath});
} else {
auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
store->registerDrvOutput(realisation);
}
logger->stopWork(); logger->stopWork();
break; break;
} }
@@ -903,15 +898,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto outputId = DrvOutput::parse(readString(from)); auto outputId = DrvOutput::parse(readString(from));
auto info = store->queryRealisation(outputId); auto info = store->queryRealisation(outputId);
logger->stopWork(); logger->stopWork();
if (GET_PROTOCOL_MINOR(clientVersion) < 31) { std::set<StorePath> outPaths;
std::set<StorePath> outPaths; if (info) outPaths.insert(info->outPath);
if (info) outPaths.insert(info->outPath); worker_proto::write(*store, to, outPaths);
worker_proto::write(*store, to, outPaths);
} else {
std::set<Realisation> realisations;
if (info) realisations.insert(*info);
worker_proto::write(*store, to, realisations);
}
break; break;
} }
+2 -2
View File
@@ -775,7 +775,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
try { try {
AutoCloseDir dir(opendir(realStoreDir.get().c_str())); AutoCloseDir dir(opendir(realStoreDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", realStoreDir); if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
/* Read the store and immediately delete all paths that /* Read the store and immediately delete all paths that
@@ -856,7 +856,7 @@ void LocalStore::autoGC(bool sync)
return std::stoll(readFile(*fakeFreeSpaceFile)); return std::stoll(readFile(*fakeFreeSpaceFile));
struct statvfs st; struct statvfs st;
if (statvfs(realStoreDir.get().c_str(), &st)) if (statvfs(realStoreDir.c_str(), &st))
throw SysError("getting filesystem info about '%s'", realStoreDir); throw SysError("getting filesystem info about '%s'", realStoreDir);
return (uint64_t) st.f_bavail * st.f_frsize; return (uint64_t) st.f_bavail * st.f_frsize;
+1 -4
View File
@@ -18,9 +18,6 @@ struct LocalFSStoreConfig : virtual StoreConfig
const PathSetting logDir{(StoreConfig*) this, false, const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"}; "log", "directory where Nix will store state"};
const PathSetting realStoreDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"};
}; };
class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
@@ -37,7 +34,7 @@ public:
/* Register a permanent GC root. */ /* Register a permanent GC root. */
Path addPermRoot(const StorePath & storePath, const Path & gcRoot); Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
virtual Path getRealStoreDir() { return realStoreDir; } virtual Path getRealStoreDir() { return storeDir; }
Path toRealPath(const Path & storePath) override Path toRealPath(const Path & storePath) override
{ {
+30 -170
View File
@@ -53,15 +53,12 @@ struct LocalStore::State::Stmts {
SQLiteStmt InvalidatePath; SQLiteStmt InvalidatePath;
SQLiteStmt AddDerivationOutput; SQLiteStmt AddDerivationOutput;
SQLiteStmt RegisterRealisedOutput; SQLiteStmt RegisterRealisedOutput;
SQLiteStmt UpdateRealisedOutput;
SQLiteStmt QueryValidDerivers; SQLiteStmt QueryValidDerivers;
SQLiteStmt QueryDerivationOutputs; SQLiteStmt QueryDerivationOutputs;
SQLiteStmt QueryRealisedOutput; SQLiteStmt QueryRealisedOutput;
SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths; SQLiteStmt QueryValidPaths;
SQLiteStmt QueryRealisationReferences;
SQLiteStmt AddRealisationReference;
}; };
int getSchema(Path schemaPath) int getSchema(Path schemaPath)
@@ -79,7 +76,7 @@ int getSchema(Path schemaPath)
void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
{ {
const int nixCASchemaVersion = 2; const int nixCASchemaVersion = 1;
int curCASchema = getSchema(schemaPath); int curCASchema = getSchema(schemaPath);
if (curCASchema != nixCASchemaVersion) { if (curCASchema != nixCASchemaVersion) {
if (curCASchema > nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) {
@@ -97,39 +94,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
#include "ca-specific-schema.sql.gen.hh" #include "ca-specific-schema.sql.gen.hh"
; ;
db.exec(schema); db.exec(schema);
curCASchema = nixCASchemaVersion;
} }
if (curCASchema < 2) {
SQLiteTxn txn(db);
// Ugly little sql dance to add a new `id` column and make it the primary key
db.exec(R"(
create table Realisations2 (
id integer primary key autoincrement not null,
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
signatures text, -- space-separated list
foreign key (outputPath) references ValidPaths(id) on delete cascade
);
insert into Realisations2 (drvPath, outputName, outputPath, signatures)
select drvPath, outputName, outputPath, signatures from Realisations;
drop table Realisations;
alter table Realisations2 rename to Realisations;
)");
db.exec(R"(
create index if not exists IndexRealisations on Realisations(drvPath, outputName);
create table if not exists RealisationsRefs (
referrer integer not null,
realisationReference integer,
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);
)");
txn.commit();
}
writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
lockFile(lockFd.get(), ltRead, true); lockFile(lockFd.get(), ltRead, true);
} }
@@ -141,6 +106,9 @@ LocalStore::LocalStore(const Params & params)
, LocalStoreConfig(params) , LocalStoreConfig(params)
, Store(params) , Store(params)
, LocalFSStore(params) , LocalFSStore(params)
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"}
, realStoreDir(realStoreDir_)
, dbDir(stateDir + "/db") , dbDir(stateDir + "/db")
, linksDir(realStoreDir + "/.links") , linksDir(realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved") , reservedPath(dbDir + "/reserved")
@@ -185,13 +153,13 @@ LocalStore::LocalStore(const Params & params)
printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
else { else {
struct stat st; struct stat st;
if (stat(realStoreDir.get().c_str(), &st)) if (stat(realStoreDir.c_str(), &st))
throw SysError("getting attributes of path '%1%'", realStoreDir); throw SysError("getting attributes of path '%1%'", realStoreDir);
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
throw SysError("changing ownership of path '%1%'", realStoreDir); throw SysError("changing ownership of path '%1%'", realStoreDir);
if (chmod(realStoreDir.get().c_str(), perm) == -1) if (chmod(realStoreDir.c_str(), perm) == -1)
throw SysError("changing permissions on path '%1%'", realStoreDir); throw SysError("changing permissions on path '%1%'", realStoreDir);
} }
} }
@@ -346,18 +314,9 @@ LocalStore::LocalStore(const Params & params)
values (?, ?, (select id from ValidPaths where path = ?), ?) values (?, ?, (select id from ValidPaths where path = ?), ?)
; ;
)"); )");
state->stmts->UpdateRealisedOutput.create(state->db,
R"(
update Realisations
set signatures = ?
where
drvPath = ? and
outputName = ?
;
)");
state->stmts->QueryRealisedOutput.create(state->db, state->stmts->QueryRealisedOutput.create(state->db,
R"( R"(
select Realisations.id, Output.path, Realisations.signatures from Realisations select Output.path, Realisations.signatures from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ? and outputName = ? where drvPath = ? and outputName = ?
; ;
@@ -369,19 +328,6 @@ LocalStore::LocalStore(const Params & params)
where drvPath = ? where drvPath = ?
; ;
)"); )");
state->stmts->QueryRealisationReferences.create(state->db,
R"(
select drvPath, outputName from Realisations
join RealisationsRefs on realisationReference = Realisations.id
where referrer = ?;
)");
state->stmts->AddRealisationReference.create(state->db,
R"(
insert or replace into RealisationsRefs (referrer, realisationReference)
values (
?,
(select id from Realisations where drvPath = ? and outputName = ?));
)");
} }
} }
@@ -491,14 +437,14 @@ void LocalStore::makeStoreWritable()
if (getuid() != 0) return; if (getuid() != 0) return;
/* Check if /nix/store is on a read-only mount. */ /* Check if /nix/store is on a read-only mount. */
struct statvfs stat; struct statvfs stat;
if (statvfs(realStoreDir.get().c_str(), &stat) != 0) if (statvfs(realStoreDir.c_str(), &stat) != 0)
throw SysError("getting info about the Nix store mount point"); throw SysError("getting info about the Nix store mount point");
if (stat.f_flag & ST_RDONLY) { if (stat.f_flag & ST_RDONLY) {
if (unshare(CLONE_NEWNS) == -1) if (unshare(CLONE_NEWNS) == -1)
throw SysError("setting up a private mount namespace"); throw SysError("setting up a private mount namespace");
if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError("remounting %1% writable", realStoreDir); throw SysError("remounting %1% writable", realStoreDir);
} }
#endif #endif
@@ -718,54 +664,14 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
void LocalStore::registerDrvOutput(const Realisation & info) void LocalStore::registerDrvOutput(const Realisation & info)
{ {
settings.requireExperimentalFeature("ca-derivations"); settings.requireExperimentalFeature("ca-derivations");
auto state(_state.lock());
retrySQLite<void>([&]() { retrySQLite<void>([&]() {
auto state(_state.lock()); state->stmts->RegisterRealisedOutput.use()
if (auto oldR = queryRealisation_(*state, info.id)) { (info.id.strHash())
if (info.isCompatibleWith(*oldR)) { (info.id.outputName)
auto combinedSignatures = oldR->signatures; (printStorePath(info.outPath))
combinedSignatures.insert(info.signatures.begin(), (concatStringsSep(" ", info.signatures))
info.signatures.end()); .exec();
state->stmts->UpdateRealisedOutput.use()
(concatStringsSep(" ", combinedSignatures))
(info.id.strHash())
(info.id.outputName)
.exec();
} else {
throw Error("Trying to register a realisation of '%s', but we already "
"have another one locally.\n"
"Local: %s\n"
"Remote: %s",
info.id.to_string(),
printStorePath(oldR->outPath),
printStorePath(info.outPath)
);
}
} else {
state->stmts->RegisterRealisedOutput.use()
(info.id.strHash())
(info.id.outputName)
(printStorePath(info.outPath))
(concatStringsSep(" ", info.signatures))
.exec();
}
uint64_t myId = state->db.getLastInsertedRowId();
for (auto & [outputId, depPath] : info.dependentRealisations) {
auto localRealisation = queryRealisationCore_(*state, outputId);
if (!localRealisation)
throw Error("unable to register the derivation '%s' as it "
"depends on the non existent '%s'",
info.id.to_string(), outputId.to_string());
if (localRealisation->second.outPath != depPath)
throw Error("unable to register the derivation '%s' as it "
"depends on a realisation of '%s' that doesn’t"
"match what we have locally",
info.id.to_string(), outputId.to_string());
state->stmts->AddRealisationReference.use()
(myId)
(outputId.strHash())
(outputId.outputName)
.exec();
}
}); });
} }
@@ -1776,69 +1682,23 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
} }
} }
std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_( std::optional<const Realisation> LocalStore::queryRealisation(
LocalStore::State & state, const DrvOutput& id) {
const DrvOutput & id) typedef std::optional<const Realisation> Ret;
{ return retrySQLite<Ret>([&]() -> Ret {
auto useQueryRealisedOutput(
state.stmts->QueryRealisedOutput.use()
(id.strHash())
(id.outputName));
if (!useQueryRealisedOutput.next())
return std::nullopt;
auto realisationDbId = useQueryRealisedOutput.getInt(0);
auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
auto signatures =
tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
return {{
realisationDbId,
Realisation{
.id = id,
.outPath = outputPath,
.signatures = signatures,
}
}};
}
std::optional<const Realisation> LocalStore::queryRealisation_(
LocalStore::State & state,
const DrvOutput & id)
{
auto maybeCore = queryRealisationCore_(state, id);
if (!maybeCore)
return std::nullopt;
auto [realisationDbId, res] = *maybeCore;
std::map<DrvOutput, StorePath> dependentRealisations;
auto useRealisationRefs(
state.stmts->QueryRealisationReferences.use()
(realisationDbId));
while (useRealisationRefs.next()) {
auto depId = DrvOutput {
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
useRealisationRefs.getStr(1),
};
auto dependentRealisation = queryRealisationCore_(state, depId);
assert(dependentRealisation); // Enforced by the db schema
auto outputPath = dependentRealisation->second.outPath;
dependentRealisations.insert({depId, outputPath});
}
res.dependentRealisations = dependentRealisations;
return { res };
}
std::optional<const Realisation>
LocalStore::queryRealisation(const DrvOutput & id)
{
return retrySQLite<std::optional<const Realisation>>([&]() {
auto state(_state.lock()); auto state(_state.lock());
return queryRealisation_(*state, id); auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
id.outputName));
if (!use.next())
return std::nullopt;
auto outputPath = parseStorePath(use.getStr(0));
auto signatures = tokenizeString<StringSet>(use.getStr(1));
return Ret{Realisation{
.id = id, .outPath = outputPath, .signatures = signatures}};
}); });
} }
FixedOutputHash LocalStore::hashCAPath( FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method, const HashType & hashType, const FileIngestionMethod & method, const HashType & hashType,
const StorePath & path) const StorePath & path)
+5 -2
View File
@@ -83,6 +83,9 @@ private:
public: public:
PathSetting realStoreDir_;
const Path realStoreDir;
const Path dbDir; const Path dbDir;
const Path linksDir; const Path linksDir;
const Path reservedPath; const Path reservedPath;
@@ -203,8 +206,6 @@ public:
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override; std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
private: private:
@@ -278,6 +279,8 @@ private:
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
void signRealisation(Realisation &); void signRealisation(Realisation &);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override; void createUser(const std::string & userName, uid_t userId) override;
// XXX: Make a generic `Store` method // XXX: Make a generic `Store` method
+6 -45
View File
@@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
res.insert(i); res.insert(i);
if (includeDerivers && path.isDerivation()) if (includeDerivers && path.isDerivation())
for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) for (auto& i : queryDerivationOutputs(path))
if (maybeOutPath && isValidPath(*maybeOutPath)) if (isValidPath(i) && queryPathInfo(i)->deriver == path)
res.insert(*maybeOutPath); res.insert(i);
return res; return res;
}; };
else else
@@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
res.insert(ref); res.insert(ref);
if (includeOutputs && path.isDerivation()) if (includeOutputs && path.isDerivation())
for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) for (auto& i : queryDerivationOutputs(path))
if (maybeOutPath && isValidPath(*maybeOutPath)) if (isValidPath(i))
res.insert(*maybeOutPath); res.insert(i);
if (includeDerivers && info->deriver && isValidPath(*info->deriver)) if (includeDerivers && info->deriver && isValidPath(*info->deriver))
res.insert(*info->deriver); res.insert(*info->deriver);
@@ -254,44 +254,5 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
}}); }});
} }
std::map<DrvOutput, StorePath> drvOutputReferences(
const std::set<Realisation> & inputRealisations,
const StorePathSet & pathReferences)
{
std::map<DrvOutput, StorePath> res;
for (const auto & input : inputRealisations) {
if (pathReferences.count(input.outPath)) {
res.insert({input.id, input.outPath});
}
}
return res;
}
std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store,
const Derivation & drv,
const StorePath & outputPath)
{
std::set<Realisation> inputRealisations;
for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
auto outputHashes =
staticOutputHashes(store, store.readDerivation(inputDrv));
for (const auto& outputName : outputNames) {
auto thisRealisation = store.queryRealisation(
DrvOutput{outputHashes.at(outputName), outputName});
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName,
store.printStorePath(inputDrv));
inputRealisations.insert(*thisRealisation);
}
}
auto info = store.queryPathInfo(outputPath);
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
} }
+1 -1
View File
@@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Make the containing directory writable, but only if it's not /* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its the store itself (we don't want or need to mess with its
permissions). */ permissions). */
bool mustToggle = dirOf(path) != realStoreDir.get(); bool mustToggle = dirOf(path) != realStoreDir;
if (mustToggle) makeWritable(dirOf(path)); if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and reset /* When we're done, make the directory read-only again and reset
-2
View File
@@ -91,8 +91,6 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
StringSet res; StringSet res;
for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i); res.insert(i);
if (!derivationHasKnownOutputPaths(drv.type()))
res.insert("ca-derivations");
return res; return res;
} }
-58
View File
@@ -1,6 +1,5 @@
#include "realisation.hh" #include "realisation.hh"
#include "store-api.hh" #include "store-api.hh"
#include "closure.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
namespace nix { namespace nix {
@@ -22,52 +21,11 @@ std::string DrvOutput::to_string() const {
return strHash() + "!" + outputName; return strHash() + "!" + outputName;
} }
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
{
std::set<Realisation> res;
Realisation::closure(store, startOutputs, res);
return res;
}
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
{
auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
std::set<Realisation> res;
for (auto& [currentDep, _] : current.dependentRealisations) {
if (auto currentRealisation = store.queryRealisation(currentDep))
res.insert(*currentRealisation);
else
throw Error(
"Unrealised derivation '%s'", currentDep.to_string());
}
return res;
};
computeClosure<Realisation>(
startOutputs, res,
[&](const Realisation& current,
std::function<void(std::promise<std::set<Realisation>>&)>
processEdges) {
std::promise<std::set<Realisation>> promise;
try {
auto res = getDeps(current);
promise.set_value(res);
} catch (...) {
promise.set_exception(std::current_exception());
}
return processEdges(promise);
});
}
nlohmann::json Realisation::toJSON() const { nlohmann::json Realisation::toJSON() const {
auto jsonDependentRealisations = nlohmann::json::object();
for (auto & [depId, depOutPath] : dependentRealisations)
jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
return nlohmann::json{ return nlohmann::json{
{"id", id.to_string()}, {"id", id.to_string()},
{"outPath", outPath.to_string()}, {"outPath", outPath.to_string()},
{"signatures", signatures}, {"signatures", signatures},
{"dependentRealisations", jsonDependentRealisations},
}; };
} }
@@ -93,16 +51,10 @@ Realisation Realisation::fromJSON(
if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
std::map <DrvOutput, StorePath> dependentRealisations;
if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end())
for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>())
dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
return Realisation{ return Realisation{
.id = DrvOutput::parse(getField("id")), .id = DrvOutput::parse(getField("id")),
.outPath = StorePath(getField("outPath")), .outPath = StorePath(getField("outPath")),
.signatures = signatures, .signatures = signatures,
.dependentRealisations = dependentRealisations,
}; };
} }
@@ -140,16 +92,6 @@ StorePath RealisedPath::path() const {
return std::visit([](auto && arg) { return arg.getPath(); }, raw); return std::visit([](auto && arg) { return arg.getPath(); }, raw);
} }
bool Realisation::isCompatibleWith(const Realisation & other) const
{
assert (id == other.id);
if (outPath == other.outPath) {
assert(dependentRealisations == other.dependentRealisations);
return true;
}
return false;
}
void RealisedPath::closure( void RealisedPath::closure(
Store& store, Store& store,
const RealisedPath::Set& startPaths, const RealisedPath::Set& startPaths,
-13
View File
@@ -28,14 +28,6 @@ struct Realisation {
StringSet signatures; StringSet signatures;
/**
* The realisations that are required for the current one to be valid.
*
* When importing this realisation, the store will first check that all its
* dependencies exist, and map to the correct output path
*/
std::map<DrvOutput, StorePath> dependentRealisations;
nlohmann::json toJSON() const; nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
@@ -44,11 +36,6 @@ struct Realisation {
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
size_t checkSignatures(const PublicKeys & publicKeys) const; size_t checkSignatures(const PublicKeys & publicKeys) const;
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res);
bool isCompatibleWith(const Realisation & other) const;
StorePath getPath() const { return outPath; } StorePath getPath() const { return outPath; }
GENERATE_CMP(Realisation, me->id, me->outPath); GENERATE_CMP(Realisation, me->id, me->outPath);
+6 -17
View File
@@ -653,12 +653,8 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
{ {
auto conn(getConnection()); auto conn(getConnection());
conn->to << wopRegisterDrvOutput; conn->to << wopRegisterDrvOutput;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { conn->to << info.id.to_string();
conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string());
conn->to << std::string(info.outPath.to_string());
} else {
worker_proto::write(*this, conn->to, info);
}
conn.processStderr(); conn.processStderr();
} }
@@ -668,17 +664,10 @@ std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput &
conn->to << wopQueryRealisation; conn->to << wopQueryRealisation;
conn->to << id.to_string(); conn->to << id.to_string();
conn.processStderr(); conn.processStderr();
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{}); if (outPaths.empty())
if (outPaths.empty()) return std::nullopt;
return std::nullopt; return {Realisation{.id = id, .outPath = *outPaths.begin()}};
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
} else {
auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
if (realisations.empty())
return std::nullopt;
return *realisations.begin();
}
} }
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs) static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
+6 -32
View File
@@ -337,13 +337,6 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
return info; return info;
} }
StringSet StoreConfig::getDefaultSystemFeatures()
{
auto res = settings.systemFeatures.get();
if (settings.isExperimentalFeatureEnabled("ca-derivations"))
res.insert("ca-derivations");
return res;
}
Store::Store(const Params & params) Store::Store(const Params & params)
: StoreConfig(params) : StoreConfig(params)
@@ -787,39 +780,20 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{ {
StorePathSet storePaths; StorePathSet storePaths;
std::set<Realisation> toplevelRealisations; std::set<Realisation> realisations;
for (auto & path : paths) { for (auto & path : paths) {
storePaths.insert(path.path()); storePaths.insert(path.path());
if (auto realisation = std::get_if<Realisation>(&path.raw)) { if (auto realisation = std::get_if<Realisation>(&path.raw)) {
settings.requireExperimentalFeature("ca-derivations"); settings.requireExperimentalFeature("ca-derivations");
toplevelRealisations.insert(*realisation); realisations.insert(*realisation);
} }
} }
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
ThreadPool pool;
try { try {
// Copy the realisation closure for (auto & realisation : realisations) {
processGraph<Realisation>( dstStore->registerDrvOutput(realisation, checkSigs);
pool, Realisation::closure(*srcStore, toplevelRealisations), }
[&](const Realisation& current) -> std::set<Realisation> { } catch (MissingExperimentalFeature & e) {
std::set<Realisation> children;
for (const auto& [drvOutput, _] : current.dependentRealisations) {
auto currentChild = srcStore->queryRealisation(drvOutput);
if (!currentChild)
throw Error(
"Incomplete realisation closure: '%s' is a "
"dependency of '%s' but isn’t registered",
drvOutput.to_string(), current.id.to_string());
children.insert(*currentChild);
}
return children;
},
[&](const Realisation& current) -> void {
dstStore->registerDrvOutput(current, checkSigs);
});
} catch (MissingExperimentalFeature& e) {
// Don't fail if the remote doesn't support CA derivations is it might // Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want // not be within our control to change that, and we might still want
// to at least copy the output paths. // to at least copy the output paths.
+1 -8
View File
@@ -180,8 +180,6 @@ struct StoreConfig : public Config
StoreConfig() = delete; StoreConfig() = delete;
StringSet getDefaultSystemFeatures();
virtual ~StoreConfig() { } virtual ~StoreConfig() { }
virtual const std::string name() = 0; virtual const std::string name() = 0;
@@ -198,7 +196,7 @@ struct StoreConfig : public Config
Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), Setting<StringSet> systemFeatures{this, settings.systemFeatures,
"system-features", "system-features",
"Optional features that the system this store builds on implements (like \"kvm\")."}; "Optional features that the system this store builds on implements (like \"kvm\")."};
@@ -866,9 +864,4 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri)
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store,
const Derivation & drv,
const StorePath & outputPath);
} }
+1 -1
View File
@@ -9,7 +9,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f #define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION (1 << 8 | 31) #define PROTOCOL_VERSION (1 << 8 | 30)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
+1 -3
View File
@@ -25,8 +25,6 @@
} }
#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
#define GENERATE_CMP(args...) \ #define GENERATE_CMP(args...) \
GENERATE_EQUAL(args) \ GENERATE_EQUAL(args) \
GENERATE_LEQ(args) \ GENERATE_LEQ(args)
GENERATE_NEQ(args)
-6
View File
@@ -387,12 +387,6 @@ static void main_nix_build(int argc, char * * argv)
if (dryRun) return; if (dryRun) return;
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto resolvedDrv = drv.tryResolve(*store);
assert(resolvedDrv && "Successfully resolved the derivation");
drv = *resolvedDrv;
}
// Set the environment. // Set the environment.
auto env = getEnv(); auto env = getEnv();
+2 -1
View File
@@ -69,7 +69,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
store2->addPermRoot(bo.path, absPath(symlink)); store2->addPermRoot(bo.path, absPath(symlink));
}, },
[&](BuiltPath::Built bfd) { [&](BuiltPath::Built bfd) {
for (auto & output : bfd.outputs) { auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
for (auto & output : builtOutputs) {
std::string symlink = outLink; std::string symlink = outLink;
if (i) symlink += fmt("-%d", i); if (i) symlink += fmt("-%d", i);
if (output.first != "out") symlink += fmt("-%s", output.first); if (output.first != "out") symlink += fmt("-%s", output.first);
+11 -20
View File
@@ -144,26 +144,17 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
/* Rehash and write the derivation. FIXME: would be nice to use /* Rehash and write the derivation. FIXME: would be nice to use
'buildDerivation', but that's privileged. */ 'buildDerivation', but that's privileged. */
drv.name += "-env"; drv.name += "-env";
for (auto & output : drv.outputs) {
output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
drv.env[output.first] = "";
}
drv.inputSrcs.insert(std::move(getEnvShPath)); drv.inputSrcs.insert(std::move(getEnvShPath));
if (settings.isExperimentalFeatureEnabled("ca-derivations")) { Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
for (auto & output : drv.outputs) {
output.second = {
.output = DerivationOutputDeferred{},
};
drv.env[output.first] = hashPlaceholder(output.first);
}
} else {
for (auto & output : drv.outputs) {
output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
drv.env[output.first] = "";
}
Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
auto outPath = store->makeOutputPath(output.first, h, drv.name); auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
drv.env[output.first] = store->printStorePath(outPath); drv.env[output.first] = store->printStorePath(outPath);
}
} }
auto shellDrvPath = writeDerivation(*store, drv); auto shellDrvPath = writeDerivation(*store, drv);
@@ -171,7 +162,8 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
/* Build the derivation. */ /* Build the derivation. */
store->buildPaths({DerivedPath::Built{shellDrvPath}}); store->buildPaths({DerivedPath::Built{shellDrvPath}});
for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
auto & [_1, optPath] = outputAndOptPath;
assert(optPath); assert(optPath);
auto & outPath = *optPath; auto & outPath = *optPath;
assert(store->isValidPath(outPath)); assert(store->isValidPath(outPath));
@@ -193,7 +185,6 @@ struct Common : InstallableCommand, MixProfile
"NIX_BUILD_TOP", "NIX_BUILD_TOP",
"NIX_ENFORCE_PURITY", "NIX_ENFORCE_PURITY",
"NIX_LOG_FD", "NIX_LOG_FD",
"NIX_REMOTE",
"PPID", "PPID",
"PWD", "PWD",
"SHELLOPTS", "SHELLOPTS",
-1
View File
@@ -15,7 +15,6 @@ R""(
``` ```
* Remove all packages: * Remove all packages:
```console ```console
# nix profile remove '.*' # nix profile remove '.*'
``` ```
+2 -2
View File
@@ -43,8 +43,8 @@ struct RunCommon : virtual Command
helper program (chrootHelper() below) to do the work. */ helper program (chrootHelper() below) to do the work. */
auto store2 = store.dynamic_pointer_cast<LocalStore>(); auto store2 = store.dynamic_pointer_cast<LocalStore>();
if (store2 && store->storeDir != store2->getRealStoreDir()) { if (store2 && store->storeDir != store2->realStoreDir) {
Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
for (auto & arg : args) helperArgs.push_back(arg); for (auto & arg : args) helperArgs.push_back(arg);
execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
+48 -34
View File
@@ -9,6 +9,7 @@
#include "shared.hh" #include "shared.hh"
#include "eval-cache.hh" #include "eval-cache.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "value.hh"
#include <regex> #include <regex>
#include <fstream> #include <fstream>
@@ -81,31 +82,46 @@ struct CmdSearch : InstallableCommand, MixJSON
uint64_t results = 0; uint64_t results = 0;
std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit; std::function<void(Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse) Value * vTmp = state->allocValue();
visit = [&](Value & current, const std::vector<Symbol> & attrPath, bool initialRecurse)
{ {
Activity act(*logger, lvlInfo, actUnknown, Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath))); fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
try { auto recurse = [&]()
auto recurse = [&]() {
{ for (auto & attrName : state->getFields(current, noPos)) {
for (const auto & attr : cursor.getAttrs()) { try {
auto cursor2 = cursor.getAttr(attr); auto attrPath2(attrPath);
auto attrPath2(attrPath); attrPath2.push_back(attrName);
attrPath2.push_back(attr); auto attrValue = state->allocValue();
visit(*cursor2, attrPath2, false); auto value_ = allocRootValue(attrValue);
} auto childField = state->lazyGetAttrField(current, {attrName}, noPos,
}; *attrValue);
if (childField != EvalState::LazyValueType::DelayedUncacheable)
visit(*attrValue, attrPath2, false);
} catch (EvalError &e) {
if (!(attrPath.size() > 0 &&
attrPath[0] == "legacyPackages"))
throw;
}
}
};
if (cursor.isDerivation()) { try {
auto maybeTypeField = state->lazyGetAttrField(current, {state->sType}, noPos, *vTmp);
if (maybeTypeField == EvalState::LazyValueType::PlainValue
&& vTmp->type() == nix::nString
&& strcmp(vTmp->string.s, "derivation") == 0) {
size_t found = 0; size_t found = 0;
DrvName name(cursor.getAttr("name")->getString()); state->getAttrFieldThrow(current, {state->sName}, noPos, *vTmp);
DrvName name(state->forceString(*vTmp));
auto aMeta = cursor.maybeGetAttr("meta"); auto hasDescription = state->getAttrField(current, {state->sMeta, state->sDescription}, noPos, *vTmp);
auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; auto description = hasDescription ? state->forceString(*vTmp) : "";
auto description = aDescription ? aDescription->getString() : "";
std::replace(description.begin(), description.end(), '\n', ' '); std::replace(description.begin(), description.end(), '\n', ' ');
auto attrPath2 = concatStringsSep(".", attrPath); auto attrPath2 = concatStringsSep(".", attrPath);
@@ -118,8 +134,8 @@ struct CmdSearch : InstallableCommand, MixJSON
std::regex_search(name.name, nameMatch, regex); std::regex_search(name.name, nameMatch, regex);
std::regex_search(description, descriptionMatch, regex); std::regex_search(description, descriptionMatch, regex);
if (!attrPathMatch.empty() if (!attrPathMatch.empty()
|| !nameMatch.empty() || !nameMatch.empty()
|| !descriptionMatch.empty()) || !descriptionMatch.empty())
found++; found++;
} }
@@ -134,39 +150,37 @@ struct CmdSearch : InstallableCommand, MixJSON
auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
if (results > 1) logger->cout(""); if (results > 1) logger->cout("");
logger->cout( logger->cout(
"* %s%s", "* %s%s",
wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
name.version != "" ? " (" + name.version + ")" : ""); name.version != "" ? " (" + name.version + ")" : "");
if (description != "") if (description != "")
logger->cout( logger->cout(
" %s", hilite(description, descriptionMatch, ANSI_NORMAL)); " %s", hilite(description, descriptionMatch, ANSI_NORMAL));
} }
} }
} } else if (
attrPath.size() == 0
else if ( || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
attrPath.size() == 0 || (attrPath[0] == "packages" && attrPath.size() <= 2))
|| (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
|| (attrPath[0] == "packages" && attrPath.size() <= 2))
recurse(); recurse();
else if (initialRecurse) else if (initialRecurse)
recurse(); recurse();
else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); auto recurseFieldInfo = state->lazyGetAttrField(current, {state->sRecurseForDerivations}, noPos, *vTmp);
if (attr && attr->getBool()) auto hasRecurse = recurseFieldInfo == EvalState::LazyValueType::PlainValue;
if (hasRecurse && state->forceBool(*vTmp, noPos))
recurse(); recurse();
} }
} catch (EvalError & e) { } catch (EvalError & e) {
if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
throw; throw;
} }
}; };
for (auto & [cursor, prefix] : installable->getCursors(*state)) for (auto & [value, pos, prefix] : installable->toValues(*state))
visit(*cursor, parseAttrPath(*state, prefix), true); visit(*value, parseAttrPath(*state, prefix), true);
if (!json && !results) if (!json && !results)
throw Error("no results for the given search term(s)!"); throw Error("no results for the given search term(s)!");
@@ -4,6 +4,4 @@ file=build-hook-ca-floating.nix
sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
CONTENT_ADDRESSED=true
source build-remote.sh source build-remote.sh
+3 -8
View File
@@ -6,17 +6,12 @@ unset NIX_STATE_DIR
function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
EXTRA_SYSTEM_FEATURES=()
if [[ -n "$CONTENT_ADDRESSED" ]]; then
EXTRA_SYSTEM_FEATURES=("ca-derivations")
fi
builders=( builders=(
# system-features will automatically be added to the outer URL, but not inner # system-features will automatically be added to the outer URL, but not inner
# remote-store URL. # remote-store URL.
"ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})" "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
"$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})" "$TEST_ROOT/machine2 - - 1 1 bar"
"ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})" "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
) )
chmod -R +w $TEST_ROOT/machine* || true chmod -R +w $TEST_ROOT/machine* || true
+3 -3
View File
@@ -1,7 +1,7 @@
source common.sh source common.sh
expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]' expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status ' nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
(.[0] | (.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and (.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs.first | match(".*multiple-outputs-a-first")) and (.outputs.first | match(".*multiple-outputs-a-first")) and
@@ -10,10 +10,10 @@ nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-statu
(.drvPath | match(".*multiple-outputs-b.drv")) and (.drvPath | match(".*multiple-outputs-b.drv")) and
(.outputs.out | match(".*multiple-outputs-b"))) (.outputs.out | match(".*multiple-outputs-b")))
' '
testNormalization () { testNormalization () {
clearStore clearStore
outPath=$(nix-build ./simple.nix --no-out-link) outPath=$(nix-build ./simple.nix)
test "$(stat -c %Y $outPath)" -eq 1 test "$(stat -c %Y $outPath)" -eq 1
} }
testNormalization testNormalization
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
# Regression test for https://github.com/NixOS/nix/issues/4858
source common.sh
sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
# Get the output path of `rootCA`, and put some garbage instead
outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"
nix-store --delete "$outPath"
touch "$outPath"
# The build should correctly remove the garbage and put the expected path instead
nix-build ./content-addressed.nix -A rootCA --no-out-link
# Rebuild it. This shouldn’t overwrite the existing path
oldInode=$(stat -c '%i' "$outPath")
nix-build ./content-addressed.nix -A rootCA --no-out-link --arg seed 2
newInode=$(stat -c '%i' "$outPath")
[[ "$oldInode" == "$newInode" ]]
@@ -1,26 +0,0 @@
source ./common.sh
sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
export REMOTE_STORE="file://$REMOTE_STORE_DIR"
rm -rf $REMOTE_STORE_DIR
clearStore
# Build dep1 and push that to the binary cache.
# This entails building (and pushing) current-time.
nix copy --to "$REMOTE_STORE" -f nondeterministic.nix dep1
clearStore
sleep 2 # To make sure that `$(date)` will be different
# Build dep2.
# As we’ve cleared the cache, we’ll have to rebuild current-time. And because
# the current time isn’t the same as before, this will yield a new (different)
# realisation
nix build -f nondeterministic.nix dep2 --no-link
# Build something that depends both on dep1 and dep2.
# If everything goes right, we should rebuild dep2 rather than fetch it from
# the cache (because that would mean duplicating `current-time` in the closure),
# and have `dep1 == dep2`.
nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link
-12
View File
@@ -1,12 +0,0 @@
#!/usr/bin/env bash
# Ensure that garbage collection works properly with ca derivations
source common.sh
sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
export NIX_TESTS_CA_BY_DEFAULT=1
cd ..
source gc.sh
-10
View File
@@ -1,10 +0,0 @@
#!/usr/bin/env bash
source common.sh
sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
CONTENT_ADDRESSED=true
cd ..
source ./nix-shell.sh
-35
View File
@@ -1,35 +0,0 @@
with import ./config.nix;
let mkCADerivation = args: mkDerivation ({
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
} // args);
in
rec {
currentTime = mkCADerivation {
name = "current-time";
buildCommand = ''
mkdir $out
echo $(date) > $out/current-time
'';
};
dep = seed: mkCADerivation {
name = "dep";
inherit seed;
buildCommand = ''
echo ${currentTime} > $out
'';
};
dep1 = dep 1;
dep2 = dep 2;
toplevel = mkCADerivation {
name = "toplevel";
buildCommand = ''
test ${dep1} == ${dep2}
touch $out
'';
};
}
-11
View File
@@ -1,11 +0,0 @@
#!/usr/bin/env bash
source common.sh
sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
export NIX_TESTS_CA_BY_DEFAULT=1
cd ..
source ./post-hook.sh
-11
View File
@@ -1,11 +0,0 @@
#!/usr/bin/env bash
source common.sh
sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
export NIX_TESTS_CA_BY_DEFAULT=1
cd ..
source ./recursive.sh
+2 -6
View File
@@ -17,15 +17,11 @@ buildDrvs () {
# Populate the remote cache # Populate the remote cache
clearStore clearStore
nix copy --to $REMOTE_STORE --file ./content-addressed.nix buildDrvs --post-build-hook ../push-to-store.sh
# Restart the build on an empty store, ensuring that we don't build # Restart the build on an empty store, ensuring that we don't build
clearStore clearStore
buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
# Check that the thing we’ve just substituted has its realisation stored
nix realisation info --file ./content-addressed.nix transitivelyDependentCA
# Check that its dependencies have it too
nix realisation info --file ./content-addressed.nix dependentCA rootCA
# Same thing, but # Same thing, but
# 1. With non-ca derivations # 1. With non-ca derivations
+1 -10
View File
@@ -1,12 +1,3 @@
let
contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1";
caArgs = if contentAddressedByDefault then {
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
} else {};
in
rec { rec {
shell = "@bash@"; shell = "@bash@";
@@ -22,6 +13,6 @@ rec {
builder = shell; builder = shell;
args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
PATH = path; PATH = path;
} // caArgs // removeAttrs args ["builder" "meta"]) } // removeAttrs args ["builder" "meta"])
// { meta = args.meta or {}; }; // { meta = args.meta or {}; };
} }
+1 -1
View File
@@ -12,7 +12,7 @@ ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo
nix-store --gc --print-roots | grep $outPath nix-store --gc --print-roots | grep $outPath
nix-store --gc --print-live | grep $outPath nix-store --gc --print-live | grep $outPath
nix-store --gc --print-dead | grep $drvPath nix-store --gc --print-dead | grep $drvPath
if nix-store --gc --print-dead | grep -E $outPath$; then false; fi if nix-store --gc --print-dead | grep $outPath; then false; fi
nix-store --gc --print-dead nix-store --gc --print-dead
-6
View File
@@ -2,7 +2,6 @@ nix_tests = \
hash.sh lang.sh add.sh simple.sh dependencies.sh \ hash.sh lang.sh add.sh simple.sh dependencies.sh \
config.sh \ config.sh \
gc.sh \ gc.sh \
ca/gc.sh \
gc-concurrent.sh \ gc-concurrent.sh \
gc-auto.sh \ gc-auto.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
@@ -39,7 +38,6 @@ nix_tests = \
search.sh \ search.sh \
nix-copy-ssh.sh \ nix-copy-ssh.sh \
post-hook.sh \ post-hook.sh \
ca/post-hook.sh \
function-trace.sh \ function-trace.sh \
recursive.sh \ recursive.sh \
describe-stores.sh \ describe-stores.sh \
@@ -47,13 +45,9 @@ nix_tests = \
build.sh \ build.sh \
compute-levels.sh \ compute-levels.sh \
ca/build.sh \ ca/build.sh \
ca/build-with-garbage-path.sh \
ca/duplicate-realisation-in-closure.sh \
ca/substitute.sh \ ca/substitute.sh \
ca/signatures.sh \ ca/signatures.sh \
ca/nix-shell.sh \
ca/nix-run.sh \ ca/nix-run.sh \
ca/recursive.sh \
ca/nix-copy.sh ca/nix-copy.sh
# parallel.sh # parallel.sh
+6 -20
View File
@@ -2,20 +2,6 @@ source common.sh
clearStore clearStore
if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
nix-shell () {
command nix-shell --arg contentAddressed true "$@"
}
nix_develop() {
nix develop --arg contentAddressed true "$@"
}
else
nix_develop() {
nix develop "$@"
}
fi
# Test nix-shell -A # Test nix-shell -A
export IMPURE_VAR=foo export IMPURE_VAR=foo
export SELECTED_IMPURE_VAR=baz export SELECTED_IMPURE_VAR=baz
@@ -55,7 +41,7 @@ output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(f
[ "$output" = "foo bar" ] [ "$output" = "foo bar" ]
# Test nix-shell shebang mode # Test nix-shell shebang mode
sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
chmod a+rx $TEST_ROOT/shell.shebang.sh chmod a+rx $TEST_ROOT/shell.shebang.sh
output=$($TEST_ROOT/shell.shebang.sh abc def) output=$($TEST_ROOT/shell.shebang.sh abc def)
@@ -63,7 +49,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def)
# Test nix-shell shebang mode again with metacharacters in the filename. # Test nix-shell shebang mode again with metacharacters in the filename.
# First word of filename is chosen to not match any file in the test root. # First word of filename is chosen to not match any file in the test root.
sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
@@ -72,7 +58,7 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
# Test nix-shell shebang mode for ruby # Test nix-shell shebang mode for ruby
# This uses a fake interpreter that returns the arguments passed # This uses a fake interpreter that returns the arguments passed
# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
chmod a+rx $TEST_ROOT/shell.shebang.rb chmod a+rx $TEST_ROOT/shell.shebang.rb
output=$($TEST_ROOT/shell.shebang.rb abc ruby) output=$($TEST_ROOT/shell.shebang.rb abc ruby)
@@ -80,20 +66,20 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
# Test nix-shell shebang mode for ruby again with metacharacters in the filename. # Test nix-shell shebang mode for ruby again with metacharacters in the filename.
# Note: fake interpreter only space-separates args without adding escapes to its output. # Note: fake interpreter only space-separates args without adding escapes to its output.
sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
# Test 'nix develop'. # Test 'nix develop'.
nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
# Ensure `nix develop -c` preserves stdin # Ensure `nix develop -c` preserves stdin
echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
# Ensure `nix develop -c` actually executes the command if stdout isn't a terminal # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo
# Test 'nix print-dev-env'. # Test 'nix print-dev-env'.
source <(nix print-dev-env -f shell.nix shellDrv) source <(nix print-dev-env -f shell.nix shellDrv)
+1 -1
View File
@@ -4,7 +4,7 @@ clearStore
rm -f $TEST_ROOT/result rm -f $TEST_ROOT/result
export REMOTE_STORE=file:$TEST_ROOT/remote_store export REMOTE_STORE=$TEST_ROOT/remote_store
# Build the dependencies and push them to the remote store # Build the dependencies and push them to the remote store
nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh
+8 -9
View File
@@ -9,9 +9,9 @@ rm -f $TEST_ROOT/result
export unreachable=$(nix store add-path ./recursive.sh) export unreachable=$(nix store add-path ./recursive.sh)
NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
with import ./config.nix; with import ./config.nix;
mkDerivation rec { mkDerivation {
name = "recursive"; name = "recursive";
dummy = builtins.toFile "dummy" "bla bla"; dummy = builtins.toFile "dummy" "bla bla";
SHELL = shell; SHELL = shell;
@@ -19,13 +19,11 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-com
# Note: this is a string without context. # Note: this is a string without context.
unreachable = builtins.getEnv "unreachable"; unreachable = builtins.getEnv "unreachable";
NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT";
requiredSystemFeatures = [ "recursive-nix" ]; requiredSystemFeatures = [ "recursive-nix" ];
buildCommand = '\'\'' buildCommand = '\'\''
mkdir $out mkdir $out
opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" opts="--experimental-features nix-command"
PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
@@ -48,15 +46,16 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-com
# Add it to our closure. # Add it to our closure.
ln -s $foobar $out/foobar ln -s $foobar $out/foobar
[[ $(nix $opts path-info --all | wc -l) -eq 4 ]] [[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
# Build a derivation. # Build a derivation.
nix $opts build -L --impure --expr '\'' nix $opts build -L --impure --expr '\''
with import ${./config.nix}; derivation {
mkDerivation {
name = "inner1"; name = "inner1";
buildCommand = "echo $fnord blaat > $out"; builder = builtins.getEnv "SHELL";
system = builtins.getEnv "system";
fnord = builtins.toFile "fnord" "fnord"; fnord = builtins.toFile "fnord" "fnord";
args = [ "-c" "echo $fnord blaat > $out" ];
} }
'\'' '\''
+2 -14
View File
@@ -1,18 +1,6 @@
{ inNixShell ? false, contentAddressed ? false }: { inNixShell ? false }:
let cfg = import ./config.nix; in with import ./config.nix;
with cfg;
let
mkDerivation =
if contentAddressed then
args: cfg.mkDerivation ({
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
} // args)
else cfg.mkDerivation;
in
let pkgs = rec { let pkgs = rec {
setupSh = builtins.toFile "setup" '' setupSh = builtins.toFile "setup" ''
+12
View File
@@ -0,0 +1,12 @@
let
# pkgs = (builtins.getFlake "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31").legacyPackages.x86_64-linux;
pkgs = import <nixpkgs> {};
in
pkgs.runCommandNoCC "foo" {
buildInputs = [
pkgs.firefox
pkgs.pandoc
# pkgs.nixosTests.xfce
];
}
"echo bar > $out"