Compare commits

..
Author SHA1 Message Date
b6e26f2b06 libstore-c: Add two new functions
- `nix_store_build_paths`
- `nix_store_query_path_info`

Co-authored-by: Tristan Ross <tristan.ross@determinate.systems>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
2025-11-25 13:10:56 -05:00
d06f221f23 libstore-c: Add new derivation and store path functions
Add several new functions to the C API:

StorePath operations:
- nix_store_path_hash: Extract the hash part from a store path
- nix_store_create_from_parts: Construct a store path from hash and name

Derivation operations:
- nix_derivation_clone: Clone a derivation
- nix_derivation_to_json: Serialize a derivation to JSON

Store operations:
- nix_store_drv_from_store_path: Load a derivation from a store path

Test the new functions, and improve documentation of some existing
functions to better distinguish them, also.

Co-authored-by: Tristan Ross <tristan.ross@determinate.systems>
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
2025-11-25 13:07:35 -05:00
23 changed files with 156 additions and 221 deletions

No files matched your search

-2
View File
@@ -79,8 +79,6 @@
# Not supported by nixfmt
''^tests/functional/lang/eval-okay-deprecate-cursed-or\.nix$''
''^tests/functional/lang/eval-okay-attrs5\.nix$''
''^tests/functional/lang/eval-fail-dynamic-attrs-inherit\.nix$''
''^tests/functional/lang/eval-fail-dynamic-attrs-inherit-2\.nix$''
# More syntax tests
# These tests, or parts of them, should have been parse-* test cases.
@@ -47,79 +47,6 @@ struct ParserLocation
}
};
/**
* This represents a string-like parse that possibly has yet to be constructed.
*
* Examples:
* "foo"
* ${"foo" + "bar"}
* "foo.bar"
* "foo-${a}"
*
* Using this type allows us to avoid construction altogether in cases where what we actually need is the string
* contents. For example in foo."bar.baz", there is no need to construct an AST node for "bar.baz", but we don't know
* that until we bubble the value up during parsing and see that it's a node in an AttrPath.
*/
class ToBeStringyExpr
{
private:
using Raw = std::variant<std::monostate, std::string_view, Expr *>;
Raw raw;
public:
ToBeStringyExpr() = default;
ToBeStringyExpr(std::string_view v)
: raw(v)
{
}
ToBeStringyExpr(Expr * expr)
: raw(expr)
{
assert(expr);
}
/**
* Visits the expression and invokes an overloaded functor object \ref f.
* If the underlying Expr has a dynamic type of ExprString the overload taking std::string_view
* is invoked.
*
* Used to consistently handle simple StringExpr ${"string"} as non-dynamic attributes.
* @see https://github.com/NixOS/nix/issues/14642
*/
template<class F>
void visit(F && f)
{
std::visit(
overloaded{
[&](std::string_view str) { f(str); },
[&](Expr * expr) {
ExprString * str = dynamic_cast<ExprString *>(expr);
if (str)
f(str->v.string_view());
else
f(expr);
},
[](std::monostate) { unreachable(); }},
raw);
}
/**
* Get or create an Expr from either an existing Expr or from a string.
* Delays the allocation or an AST node in case the parser only cares about string contents.
*/
Expr * toExpr(Exprs & exprs)
{
return std::visit(
overloaded{
[&](std::string_view str) -> Expr * { return exprs.add<ExprString>(exprs.alloc, str); },
[&](Expr * expr) { return expr; },
[](std::monostate) -> Expr * { unreachable(); }},
raw);
}
};
struct LexerState
{
/**
+24 -19
View File
@@ -138,7 +138,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%type <std::vector<std::pair<PosIdx, Expr *>>> string_parts_interpolated
%type <std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>> ind_string_parts
%type <Expr *> path_start
%type <ToBeStringyExpr> string_parts string_attr
%type <std::variant<Expr *, std::string_view>> string_parts string_attr
%type <StringToken> attr
%token <StringToken> ID
%token <StringToken> STR IND_STR
@@ -297,7 +297,12 @@ expr_simple
}
| INT_LIT { $$ = state->exprs.add<ExprInt>($1); }
| FLOAT_LIT { $$ = state->exprs.add<ExprFloat>($1); }
| '"' string_parts '"' { $$ = $2.toExpr(state->exprs); }
| '"' string_parts '"' {
std::visit(overloaded{
[&](std::string_view str) { $$ = state->exprs.add<ExprString>(state->exprs.alloc, str); },
[&](Expr * expr) { $$ = expr; }},
$2);
}
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = state->stripIndentation(CUR_POS, $2);
}
@@ -337,9 +342,9 @@ expr_simple
;
string_parts
: STR { $$ = {$1}; }
| string_parts_interpolated { $$ = {state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1)}; }
| { $$ = {std::string_view()}; }
: STR { $$ = $1; }
| string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1); }
| { $$ = std::string_view(); }
;
string_parts_interpolated
@@ -442,15 +447,15 @@ attrs
: attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); }
| attrs string_attr
{ $$ = std::move($1);
$2.visit(overloaded{
std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); },
[&](Expr * expr) {
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}}
);
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}
}, $2);
}
| { }
;
@@ -459,17 +464,17 @@ attrpath
: attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); }
| attrpath '.' string_attr
{ $$ = std::move($1);
$3.visit(overloaded{
std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }}
);
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($3));
}
| attr { $$.emplace_back(state->symbols.create($1)); }
| string_attr
{ $1.visit(overloaded{
{ std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }}
);
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($1));
}
;
@@ -480,7 +485,7 @@ attr
string_attr
: '"' string_parts '"' { $$ = std::move($2); }
| DOLLAR_CURLY expr '}' { $$ = {$2}; }
| DOLLAR_CURLY expr '}' { $$ = $2; }
;
list
+9 -13
View File
@@ -470,8 +470,7 @@ public:
std::string res;
auto renderActivity =
[&] [[nodiscard]] (
ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
[&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
auto & act = state.activitiesByType[type];
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
for (auto & j : act.its) {
@@ -515,7 +514,7 @@ public:
return s;
};
auto renderSizeActivity = [&] [[nodiscard]] (ActivityType type, const std::string & itemFmt = "%s") {
auto renderSizeActivity = [&](ActivityType type, const std::string & itemFmt = "%s") {
auto & act = state.activitiesByType[type];
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
for (auto & j : act.its) {
@@ -574,17 +573,14 @@ public:
return s;
};
auto maybeAppendToResult = [&](std::string_view s) {
if (s.empty())
return;
if (!res.empty())
res += ", ";
res += s;
};
auto showActivity =
[&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
maybeAppendToResult(renderActivity(type, itemFmt, numberFmt, unit));
auto s = renderActivity(type, itemFmt, numberFmt, unit);
if (s.empty())
return;
if (!res.empty())
res += ", ";
res += s;
};
showActivity(actBuilds, "%s built");
@@ -606,7 +602,7 @@ public:
}
}
maybeAppendToResult(renderSizeActivity(actFileTransfer, "%s DL"));
renderSizeActivity(actFileTransfer, "%s DL");
{
auto s = renderActivity(actOptimiseStore, "%s paths optimised");
+51 -1
View File
@@ -226,7 +226,7 @@ StorePath * nix_store_path_clone(const StorePath * p)
}
}
} // extern "C"
}
template<size_t S>
static auto to_cpp_array(const uint8_t (&r)[S])
@@ -338,4 +338,54 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
NIXC_CATCH_ERRS_NULL
}
nix_err nix_store_query_path_info(
nix_c_context * context,
Store * store,
const StorePath * store_path,
void * userdata,
nix_get_string_callback callback)
{
if (context)
context->last_err_code = NIX_OK;
try {
auto info = store->ptr->queryPathInfo(store_path->path);
if (callback) {
auto result = info->toJSON(&*store->ptr, true).dump();
callback(result.data(), result.size(), userdata);
}
}
NIXC_CATCH_ERRS
}
nix_err nix_store_build_paths(
nix_c_context * context,
Store * store,
const StorePath ** store_paths,
unsigned int num_store_paths,
void (*callback)(void * userdata, const char * path, const char * result),
void * userdata)
{
if (context)
context->last_err_code = NIX_OK;
try {
std::span<const StorePath * const> paths_span(store_paths, num_store_paths);
std::vector<nix::DerivedPath> derived_paths;
for (const StorePath * store_path : paths_span) {
derived_paths.push_back(nix::SingleDerivedPath::Opaque{store_path->path});
}
auto results = store->ptr->buildPathsWithResults(derived_paths);
for (auto & result : results) {
if (callback) {
callback(
userdata,
result.path.to_string(store->ptr->config).c_str(),
static_cast<nlohmann::json>(result).dump().c_str());
}
}
}
NIXC_CATCH_ERRS
}
} // extern "C"
+40
View File
@@ -259,6 +259,46 @@ nix_err nix_store_get_fs_closure(
*/
nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path);
/**
* @brief Queries for the nix store path info JSON.
*
* @param[out] context Optional, stores error information
* @param[in] store nix store reference
* @param[in] path A store path
* @param[in] userdata The data to pass to the callback
* @param[in] callback Called for when the path info is resolved
*/
nix_err nix_store_query_path_info(
nix_c_context * context,
Store * store,
const StorePath * store_path,
void * userdata,
nix_get_string_callback callback);
/**
* @brief Builds the paths, if they are a derivation then they get built.
*
* @note Path and result for the callback only exist for the lifetime of
* the call. Result is a string containing the build result in JSON.
*
* @param[out] context Optional, stores error information
* @param[in] store nix store reference
* @param[in] store_paths Pointer to list of nix store paths
* @param[in] num_store_paths Number of nix store paths
* @param[in] callback The callback to trigger for each build result. Called with:
* - userdata: user-provided data
* - path: the path that was built (string, only valid for duration of callback)
* - result: JSON string of the build result (only valid for duration of callback)
* @param[in] userdata User data to pass to the callback
*/
nix_err nix_store_build_paths(
nix_c_context * context,
Store * store,
const StorePath ** store_paths,
unsigned int num_store_paths,
void (*callback)(void * userdata, const char * path, const char * result),
void * userdata);
// cffi end
#ifdef __cplusplus
}
+6 -6
View File
@@ -83,7 +83,7 @@ bool PathLocks::lockPaths(const std::set<std::filesystem::path> & paths, const s
checkInterrupt();
std::filesystem::path lockPath = path + ".lock";
debug("locking path %1%", path);
debug("locking path '%1%'", path);
AutoCloseFD fd;
@@ -106,19 +106,19 @@ bool PathLocks::lockPaths(const std::set<std::filesystem::path> & paths, const s
}
}
debug("lock acquired on %1%", lockPath);
debug("lock acquired on '%1%'", lockPath);
/* Check that the lock file hasn't become stale (i.e.,
hasn't been unlinked). */
struct stat st;
if (fstat(fd.get(), &st) == -1)
throw SysError("statting lock file %1%", lockPath);
throw SysError("statting lock file '%1%'", lockPath);
if (st.st_size != 0)
/* This lock file has been unlinked, so we're holding
a lock on a deleted file. This means that other
processes may create and acquire a lock on
`lockPath', and proceed. So we must retry. */
debug("open lock file %1% has become stale", lockPath);
debug("open lock file '%1%' has become stale", lockPath);
else
break;
}
@@ -137,9 +137,9 @@ void PathLocks::unlock()
deleteLockFile(i.second, i.first);
if (close(i.first) == -1)
printError("error (ignored): cannot close lock file on %1%", i.second);
printError("error (ignored): cannot close lock file on '%1%'", i.second);
debug("lock released on %1%", i.second);
debug("lock released on '%1%'", i.second);
}
fds.clear();
+6 -6
View File
@@ -28,9 +28,9 @@ void PathLocks::unlock()
deleteLockFile(i.second, i.first);
if (CloseHandle(i.first) == -1)
printError("error (ignored): cannot close lock file on %1%", i.second);
printError("error (ignored): cannot close lock file on '%1%'", i.second);
debug("lock released on %1%", i.second);
debug("lock released on '%1%'", i.second);
}
fds.clear();
@@ -111,7 +111,7 @@ bool PathLocks::lockPaths(const std::set<std::filesystem::path> & paths, const s
checkInterrupt();
std::filesystem::path lockPath = path;
lockPath += L".lock";
debug("locking path %1%", path);
debug("locking path '%1%'", path);
AutoCloseFD fd;
@@ -128,13 +128,13 @@ bool PathLocks::lockPaths(const std::set<std::filesystem::path> & paths, const s
}
}
debug("lock acquired on %1%", lockPath);
debug("lock acquired on '%1%'", lockPath);
struct _stat st;
if (_fstat(fromDescriptorReadOnly(fd.get()), &st) == -1)
throw SysError("statting lock file %1%", lockPath);
throw SysError("statting lock file '%1%'", lockPath);
if (st.st_size != 0)
debug("open lock file %1% has become stale", lockPath);
debug("open lock file '%1%' has become stale", lockPath);
else
break;
}
+19 -17
View File
@@ -272,24 +272,26 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
drv.name += "-env";
drv.env.emplace("name", drv.name);
drv.inputSrcs.insert(std::move(getEnvShPath));
for (auto & [outputName, output] : drv.outputs) {
std::visit(
overloaded{
[&](const DerivationOutput::InputAddressed &) {
output = DerivationOutput::Deferred{};
drv.env[outputName] = "";
},
[&](const DerivationOutput::CAFixed &) {
output = DerivationOutput::Deferred{};
drv.env[outputName] = "";
},
[&](const auto &) {
// Do nothing for other types (CAFloating, Deferred, Impure)
},
},
output.raw);
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
for (auto & output : drv.outputs) {
output.second = DerivationOutput::Deferred{}, drv.env[output.first] = hashPlaceholder(output.first);
}
} else {
for (auto & output : drv.outputs) {
output.second = DerivationOutput::Deferred{};
drv.env[output.first] = "";
}
auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
for (auto & output : drv.outputs) {
Hash h = hashesModulo.hashes.at(output.first);
auto outPath = store->makeOutputPath(output.first, h, drv.name);
output.second = DerivationOutput::InputAddressed{
.path = outPath,
};
drv.env[output.first] = store->printStorePath(outPath);
}
}
drv.fillInOutputPaths(*evalStore);
auto shellDrvPath = writeDerivation(*evalStore, drv);
@@ -1,6 +0,0 @@
error: dynamic attributes not allowed in inherit
at /pwd/lang/eval-fail-dynamic-attrs-inherit-2.nix:5:15:
4| {
5| inherit (a) ${"b" + ""};
| ^
6| }
@@ -1,6 +0,0 @@
let
a.b = 1;
in
{
inherit (a) ${"b" + ""};
}
@@ -1,6 +0,0 @@
error: dynamic attributes not allowed in inherit
at /pwd/lang/eval-fail-dynamic-attrs-inherit.nix:5:11:
4| {
5| inherit ${"a" + ""};
| ^
6| }
@@ -1,6 +0,0 @@
let
a = 1;
in
{
inherit ${"a" + ""};
}
@@ -1,5 +0,0 @@
error: dynamic attributes not allowed in let
at /pwd/lang/eval-fail-dynamic-attrs-let-2.nix:1:1:
1| let
| ^
2| ${"${"a"}"} = 1;
@@ -1,4 +0,0 @@
let
${"${"a"}"} = 1;
in
a
@@ -1,5 +0,0 @@
error: dynamic attributes not allowed in let
at /pwd/lang/eval-fail-dynamic-attrs-let-3.nix:1:1:
1| let
| ^
2| "${"a"}" = 1;
@@ -1,4 +0,0 @@
let
"${"a"}" = 1;
in
a
@@ -1,5 +0,0 @@
error: dynamic attributes not allowed in let
at /pwd/lang/eval-fail-dynamic-attrs-let.nix:1:1:
1| let
| ^
2| ${"a" + ""} = 1;
@@ -1,4 +0,0 @@
let
${"a" + ""} = 1;
in
a
@@ -1 +0,0 @@
{ a = 1; attrs = { b = 1; c = 1; d = 1; }; b = 1; c = 1; d = 1; }
@@ -1,14 +0,0 @@
# dynamic attrs are not generally allowed in `let`, and inherit, but they are if they only contain a string
let
${"a"} = 1;
attrs = rec {
b = c;
${"c"} = d;
d = a;
};
in
{
inherit ${"a"};
inherit attrs;
inherit (attrs) ${"b"} ${"c"} d;
}
+1 -8
View File
@@ -175,7 +175,7 @@ cat >"$TEST_ROOT"/marco/polo/default.nix <<EOF
(import $TEST_ROOT/lookup-test/shell.nix {}).polo
EOF
chmod a+x "$TEST_ROOT"/marco/polo/default.nix
(cd "$TEST_ROOT"/marco && ./polo/default.nix < /dev/null | grepQuiet "Polo")
(cd "$TEST_ROOT"/marco && ./polo/default.nix | grepQuiet "Polo")
# https://github.com/NixOS/nix/issues/11892
mkdir "$TEST_ROOT"/issue-11892
@@ -279,10 +279,3 @@ assert (!(args ? inNixShell));
(import $shellDotNix { }).shellDrv
EOF
nix-shell "$TEST_ROOT"/shell-ellipsis.nix --run "true"
# FIXME unclear why this (newly made) test is failing in this case.
if ! isTestOnNixOS; then
# `nix develop` should also work with fixed-output derivations
# shellcheck disable=SC2016
nix develop -f "$shellDotNix" fixed -c bash -c '[[ -n $stdenv ]]'
fi
-10
View File
@@ -84,16 +84,6 @@ let
'';
};
# Shells should also work with fixed-output derivations
fixed = mkDerivation {
name = "fixed";
FOO = "was a fixed-output derivation";
outputHash = "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik";
outputHashMode = "recursive";
outputHashAlgo = "sha256";
outputs = [ "out" ];
};
# Used by nix-shell -p
runCommand =
name: args: buildCommand: