Compare commits

..
Author SHA1 Message Date
John Ericson adcbf3e19a WIP strip strings at compile time 2025-11-17 14:32:33 -05:00
166 changed files with 1199 additions and 3029 deletions

No files matched your search

+1 -1
View File
@@ -174,7 +174,7 @@ jobs:
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4 - uses: cachix/install-nix-action@7ec16f2c061ab07b235a7245e06ed46fe9a1cab6 # v31.8.3
if: ${{ !matrix.experimental-installer }} if: ${{ !matrix.experimental-installer }}
with: with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
-1
View File
@@ -37,7 +37,6 @@ mkMesonDerivation (finalAttrs: {
(fileset.unions [ (fileset.unions [
../../.version ../../.version
# For example JSON # For example JSON
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash ../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/store-path
-12
View File
@@ -1,12 +0,0 @@
---
synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
prs: [14614]
issues: [11728]
---
The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
-1
View File
@@ -121,7 +121,6 @@
- [Architecture and Design](architecture/architecture.md) - [Architecture and Design](architecture/architecture.md)
- [Formats and Protocols](protocols/index.md) - [Formats and Protocols](protocols/index.md)
- [JSON Formats](protocols/json/index.md) - [JSON Formats](protocols/json/index.md)
- [File System Object](protocols/json/file-system-object.md)
- [Hash](protocols/json/hash.md) - [Hash](protocols/json/hash.md)
- [Content Address](protocols/json/content-address.md) - [Content Address](protocols/json/content-address.md)
- [Store Path](protocols/json/store-path.md) - [Store Path](protocols/json/store-path.md)
+3 -3
View File
@@ -36,7 +36,7 @@ to a temporary location. The tarball must include a single top-level
directory containing at least a file named `default.nix`. directory containing at least a file named `default.nix`.
`nix-build` is essentially a wrapper around `nix-build` is essentially a wrapper around
[`nix-instantiate`](./nix-instantiate.md) (to translate a high-level Nix [`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
expression to a low-level [store derivation]) and [`nix-store expression to a low-level [store derivation]) and [`nix-store
--realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
derivation). derivation).
@@ -52,8 +52,8 @@ derivation).
# Options # Options
All options not listed here are passed to All options not listed here are passed to
[`nix-store --realise`](./nix-store/realise.md), [`nix-store --realise`](nix-store/realise.md),
except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](./nix-instantiate.md). except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
- <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span> - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
@@ -32,7 +32,7 @@ standard input.
- `--add-root` *path* - `--add-root` *path*
See the [corresponding option](./nix-store.md) in `nix-store`. See the [corresponding option](nix-store.md) in `nix-store`.
- `--parse` - `--parse`
+1 -1
View File
@@ -15,7 +15,7 @@ In the development shell, set the `mesonBuildType` environment variable to `debu
Then, proceed to build Nix as described in [Building Nix](./building.md). Then, proceed to build Nix as described in [Building Nix](./building.md).
This will build Nix with debug symbols, which are essential for effective debugging. This will build Nix with debug symbols, which are essential for effective debugging.
It is also possible to build without optimization for faster build: It is also possible to build without debugging for faster build:
```console ```console
[nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify) [nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify)
@@ -1,21 +0,0 @@
{{#include file-system-object-v1-fixed.md}}
## Examples
### Simple
```json
{{#include schema/file-system-object-v1/simple.json}}
```
### Complex
```json
{{#include schema/file-system-object-v1/complex.json}}
```
<!-- need to convert YAML to JSON first
## Raw Schema
[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
-->
@@ -11,7 +11,6 @@ s/\\`/`/g
# #
# As we have more such relative links, more replacements of this nature # As we have more such relative links, more replacements of this nature
# should appear below. # should appear below.
s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
@@ -9,7 +9,6 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
json_schema_config = files('json-schema-for-humans-config.yaml') json_schema_config = files('json-schema-for-humans-config.yaml')
schemas = [ schemas = [
'file-system-object-v1',
'hash-v1', 'hash-v1',
'content-address-v1', 'content-address-v1',
'store-path-v1', 'store-path-v1',
@@ -1 +0,0 @@
../../../../../../src/libutil-tests/data/memory-source-accessor
@@ -1,71 +0,0 @@
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
title: File System Object
description: |
This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
The schema is recursive because file system objects contain other file system objects.
type: object
required: ["type"]
properties:
type:
type: string
enum: ["regular", "symlink", "directory"]
# Enforce conditional structure based on `type`
anyOf:
- $ref: "#/$defs/regular"
required: ["type", "contents"]
- $ref: "#/$defs/directory"
required: ["type", "entries"]
- $ref: "#/$defs/symlink"
required: ["type", "target"]
"$defs":
regular:
title: Regular File
description: |
See [Regular File](@docroot@/store/file-system-object.md#regular) in the manual for details.
required: ["contents"]
properties:
type:
const: "regular"
contents:
type: string
description: File contents
executable:
type: boolean
description: Whether the file is executable.
default: false
additionalProperties: false
directory:
title: Directory
description: |
See [Directory](@docroot@/store/file-system-object.md#directory) in the manual for details.
required: ["entries"]
properties:
type:
const: "directory"
entries:
type: object
description: |
Map of names to nested file system objects (for type=directory)
additionalProperties:
$ref: "#"
additionalProperties: false
symlink:
title: Symbolic Link
description: |
See [Symbolic Link](@docroot@/store/file-system-object.md#symlink) in the manual for details.
required: ["target"]
properties:
type:
const: "symlink"
target:
type: string
description: Target path of the symlink.
additionalProperties: false
@@ -3,23 +3,19 @@
Nix uses a simplified model of the file system, which consists of file system objects. Nix uses a simplified model of the file system, which consists of file system objects.
Every file system object is one of the following: Every file system object is one of the following:
- [**Regular File**]{#regular} - File
- A possibly empty sequence of bytes for contents - A possibly empty sequence of bytes for contents
- A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
- [**Directory**]{#directory} - Directory
Mapping of names to child file system objects Mapping of names to child file system objects
- [**Symbolic link**]{#symlink} - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
An arbitrary string, known as the *target* of the symlink. An arbitrary string.
Nix does not assign any semantics to symbolic links.
In general, Nix does not assign any semantics to symbolic links.
Certain operations however, may make additional assumptions and attempt to use the target to find another file system object.
> See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept.
File system objects and their children form a tree. File system objects and their children form a tree.
A bare file or symlink can be a root file system object. A bare file or symlink can be a root file system object.
+16 -22
View File
@@ -130,19 +130,15 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
availableComponents = lib.filterAttrs (
k: v: lib.meta.availableOn pkgs.hostPlatform v
) allComponents;
activeComponents = buildInputsClosureCond isInternal ( activeComponents = buildInputsClosureCond isInternal (
lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents) lib.attrValues (finalAttrs.passthru.config.getComponents allComponents)
); );
allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2; allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
internalDrvs = byDrvPath ( internalDrvs = byDrvPath (
# Drop the attr names (not present in buildInputs anyway) # Drop the attr names (not present in buildInputs anyway)
lib.attrValues availableComponents lib.attrValues allComponents
++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents) ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents)
); );
isInternal = isInternal =
@@ -191,19 +187,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
); );
small = small =
(finalAttrs.finalPackage.withActiveComponents ( (finalAttrs.finalPackage.withActiveComponents (c: {
c: inherit (c)
lib.intersectAttrs (lib.genAttrs [ nix-cli
"nix-cli" nix-util-tests
"nix-util-tests" nix-store-tests
"nix-store-tests" nix-expr-tests
"nix-expr-tests" nix-fetchers-tests
"nix-fetchers-tests" nix-flake-tests
"nix-flake-tests" nix-functional-tests
"nix-functional-tests" # Currently required
"nix-perl-bindings" nix-perl-bindings
] (_: null)) c ;
)).overrideAttrs })).overrideAttrs
(o: { (o: {
mesonFlags = o.mesonFlags ++ [ mesonFlags = o.mesonFlags ++ [
# TODO: infer from activeComponents or vice versa # TODO: infer from activeComponents or vice versa
@@ -273,8 +269,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
CXX_LD = "mold"; CXX_LD = "mold";
}; };
dontUseCmakeConfigure = true;
mesonFlags = mesonFlags =
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
@@ -1 +0,0 @@
../../src/libutil-tests/data/memory-source-accessor
-8
View File
@@ -20,14 +20,6 @@ schema_dir = meson.current_source_dir() / 'schema'
# Get all example files # Get all example files
schemas = [ schemas = [
{
'stem' : 'file-system-object',
'schema' : schema_dir / 'file-system-object-v1.yaml',
'files' : [
'simple.json',
'complex.json',
],
},
{ {
'stem' : 'hash', 'stem' : 'hash',
'schema' : schema_dir / 'hash-v1.yaml', 'schema' : schema_dir / 'hash-v1.yaml',
-1
View File
@@ -20,7 +20,6 @@ mkMesonDerivation (finalAttrs: {
fileset = lib.fileset.unions [ fileset = lib.fileset.unions [
../../.version ../../.version
../../doc/manual/source/protocols/json/schema ../../doc/manual/source/protocols/json/schema
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash ../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/store-path
+1 -1
View File
@@ -65,6 +65,6 @@ mkMesonDerivation (finalAttrs: {
''; '';
meta = { meta = {
platforms = lib.platforms.unix; platforms = lib.platforms.all;
}; };
}) })
+7 -8
View File
@@ -34,7 +34,7 @@ EvalSettings evalSettings{
auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false);
debug("fetching flake search path element '%s''", rest); debug("fetching flake search path element '%s''", rest);
auto [accessor, lockedRef] = auto [accessor, lockedRef] =
flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store); flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
auto storePath = nix::fetchToStore( auto storePath = nix::fetchToStore(
state.fetchSettings, state.fetchSettings,
*state.store, *state.store,
@@ -132,7 +132,7 @@ MixEvalArgs::MixEvalArgs()
fetchers::Attrs extraAttrs; fetchers::Attrs extraAttrs;
if (to.subdir != "") if (to.subdir != "")
extraAttrs["dir"] = to.subdir; extraAttrs["dir"] = to.subdir;
fetchers::overrideRegistry(from.input, to.input, extraAttrs); fetchers::overrideRegistry(fetchSettings, from.input, to.input, extraAttrs);
}}, }},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(completions, openStore(), prefix); completeFlakeRef(completions, openStore(), prefix);
@@ -168,9 +168,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
? state.rootPath(absPath(getCommandBaseDir())) ? state.rootPath(absPath(getCommandBaseDir()))
: state.rootPath("."))); : state.rootPath(".")));
}, },
[&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); }, [&](const AutoArgString & arg) { v->mkString(arg.s); },
[&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string()), state.mem); }, [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string())); },
[&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO), state.mem); }}, [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); }},
arg); arg);
res.insert(state.symbols.create(name), v); res.insert(state.symbols.create(name), v);
} }
@@ -180,7 +180,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir) SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
{ {
if (EvalSettings::isPseudoUrl(s)) { if (EvalSettings::isPseudoUrl(s)) {
auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s)); auto accessor = fetchers::downloadTarball(state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy);
return state.storePath(storePath); return state.storePath(storePath);
} }
@@ -188,8 +188,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
else if (hasPrefix(s, "flake:")) { else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto [accessor, lockedRef] = auto [accessor, lockedRef] = flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
auto storePath = nix::fetchToStore( auto storePath = nix::fetchToStore(
state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
state.allowPath(storePath); state.allowPath(storePath);
+2 -1
View File
@@ -185,6 +185,7 @@ MixFlakeOptions::MixFlakeOptions()
} }
overrideRegistry( overrideRegistry(
fetchSettings,
fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}), fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
input3->lockedRef.input, input3->lockedRef.input,
extraAttrs); extraAttrs);
@@ -409,7 +410,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
Args::completeDir(completions, 0, prefix); Args::completeDir(completions, 0, prefix);
/* Look for registry entries that match the prefix. */ /* Look for registry entries that match the prefix. */
for (auto & registry : fetchers::getRegistries(fetchSettings, *store)) { for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
for (auto & entry : registry->entries) { for (auto & entry : registry->entries) {
auto from = entry.from.to_string(); auto from = entry.from.to_string();
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
+1 -1
View File
@@ -739,7 +739,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true); auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings)) if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings))
throw Error("cannot use ':load-flake' on unlocked flake reference '%s' (use --impure to override)", flakeRefS); throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
Value v; Value v;
+8 -14
View File
@@ -69,8 +69,8 @@ nix_err nix_expr_eval_from_string(
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
state->state.eval(parsedExpr, *value->value); state->state.eval(parsedExpr, value->value);
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -80,8 +80,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos); state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -91,15 +91,9 @@ nix_err nix_value_call_multi(
{ {
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
std::vector<nix::Value *> internal_args;
internal_args.reserve(nargs);
for (size_t i = 0; i < nargs; i++)
internal_args.push_back(args[i]->value);
try { try {
state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos); state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos);
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -109,7 +103,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -119,7 +113,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
state->state.forceValueDeep(*value->value); state->state.forceValueDeep(value->value);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
+1 -7
View File
@@ -39,13 +39,7 @@ struct ListBuilder
struct nix_value struct nix_value
{ {
nix::Value * value; nix::Value value;
/**
* As we move to a managed heap, we need EvalMemory in more places. Ideally, we would take in EvalState or
* EvalMemory as an argument when we need it, but we don't want to make changes to the stable C api, so we stuff it
* into the nix_value that will get passed in to the relevant functions.
*/
nix::EvalMemory * mem;
}; };
struct nix_string_return struct nix_string_return
+25 -37
View File
@@ -20,7 +20,7 @@ static const nix::Value & check_value_not_null(const nix_value * value)
if (!value) { if (!value) {
throw std::runtime_error("nix_value is null"); throw std::runtime_error("nix_value is null");
} }
return *value->value; return *((const nix::Value *) value);
} }
static nix::Value & check_value_not_null(nix_value * value) static nix::Value & check_value_not_null(nix_value * value)
@@ -28,7 +28,7 @@ static nix::Value & check_value_not_null(nix_value * value)
if (!value) { if (!value) {
throw std::runtime_error("nix_value is null"); throw std::runtime_error("nix_value is null");
} }
return *value->value; return value->value;
} }
static const nix::Value & check_value_in(const nix_value * value) static const nix::Value & check_value_in(const nix_value * value)
@@ -58,14 +58,9 @@ static nix::Value & check_value_out(nix_value * value)
return v; return v;
} }
static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem) static inline nix_value * as_nix_value_ptr(nix::Value * v)
{ {
nix_value * ret = new (mem.allocBytes(sizeof(nix_value))) nix_value{ return reinterpret_cast<nix_value *>(v);
.value = v,
.mem = &mem,
};
nix_gc_incref(nullptr, ret);
return ret;
} }
/** /**
@@ -74,13 +69,7 @@ static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
* Deals with errors and converts arguments from C++ into C types. * Deals with errors and converts arguments from C++ into C types.
*/ */
static void nix_c_primop_wrapper( static void nix_c_primop_wrapper(
PrimOpFun f, PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
void * userdata,
int arity,
nix::EvalState & state,
const nix::PosIdx pos,
nix::Value ** args,
nix::Value & v)
{ {
nix_c_context ctx; nix_c_context ctx;
@@ -96,15 +85,8 @@ static void nix_c_primop_wrapper(
// ok because we don't see a need for this yet (e.g. inspecting thunks, // ok because we don't see a need for this yet (e.g. inspecting thunks,
// or maybe something to make blackholes work better; we don't know). // or maybe something to make blackholes work better; we don't know).
nix::Value vTmp; nix::Value vTmp;
nix_value * vTmpPtr = new_nix_value(&vTmp, state.mem);
std::vector<nix_value *> external_args; f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
external_args.reserve(arity);
for (int i = 0; i < arity; i++) {
nix_value * external_arg = new_nix_value(args[i], state.mem);
external_args.push_back(external_arg);
}
f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr);
if (ctx.last_err_code != NIX_OK) { if (ctx.last_err_code != NIX_OK) {
/* TODO: Throw different errors depending on the error code */ /* TODO: Throw different errors depending on the error code */
@@ -153,7 +135,7 @@ PrimOp * nix_alloc_primop(
.args = {}, .args = {},
.arity = (size_t) arity, .arity = (size_t) arity,
.doc = doc, .doc = doc,
.fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)}; .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
if (args) if (args)
for (size_t i = 0; args[i]; i++) for (size_t i = 0; args[i]; i++)
p->args.emplace_back(*args); p->args.emplace_back(*args);
@@ -178,7 +160,8 @@ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
nix_value * res = new_nix_value(state->state.allocValue(), state->state.mem); nix_value * res = as_nix_value_ptr(state->state.allocValue());
nix_gc_incref(nullptr, res);
return res; return res;
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
@@ -348,10 +331,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
return nullptr; return nullptr;
} }
auto * p = v.listView()[ix]; auto * p = v.listView()[ix];
if (p == nullptr) nix_gc_incref(nullptr, p);
return nullptr; if (p != nullptr)
state->state.forceValue(*p, nix::noPos); state->state.forceValue(*p, nix::noPos);
return new_nix_value(p, state->state.mem); return as_nix_value_ptr(p);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -369,8 +352,9 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
return nullptr; return nullptr;
} }
auto * p = v.listView()[ix]; auto * p = v.listView()[ix];
nix_gc_incref(nullptr, p);
// Note: intentionally NOT calling forceValue() to keep the element lazy // Note: intentionally NOT calling forceValue() to keep the element lazy
return new_nix_value(p, state->state.mem); return as_nix_value_ptr(p);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -385,8 +369,9 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
nix::Symbol s = state->state.symbols.create(name); nix::Symbol s = state->state.symbols.create(name);
auto attr = v.attrs()->get(s); auto attr = v.attrs()->get(s);
if (attr) { if (attr) {
nix_gc_incref(nullptr, attr->value);
state->state.forceValue(*attr->value, nix::noPos); state->state.forceValue(*attr->value, nix::noPos);
return new_nix_value(attr->value, state->state.mem); return as_nix_value_ptr(attr->value);
} }
nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
return nullptr; return nullptr;
@@ -405,8 +390,9 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
nix::Symbol s = state->state.symbols.create(name); nix::Symbol s = state->state.symbols.create(name);
auto attr = v.attrs()->get(s); auto attr = v.attrs()->get(s);
if (attr) { if (attr) {
nix_gc_incref(nullptr, attr->value);
// Note: intentionally NOT calling forceValue() to keep the attribute lazy // Note: intentionally NOT calling forceValue() to keep the attribute lazy
return new_nix_value(attr->value, state->state.mem); return as_nix_value_ptr(attr->value);
} }
nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
return nullptr; return nullptr;
@@ -454,8 +440,9 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
} }
const nix::Attr & a = (*v.attrs())[i]; const nix::Attr & a = (*v.attrs())[i];
*name = state->state.symbols[a.name].c_str(); *name = state->state.symbols[a.name].c_str();
nix_gc_incref(nullptr, a.value);
state->state.forceValue(*a.value, nix::noPos); state->state.forceValue(*a.value, nix::noPos);
return new_nix_value(a.value, state->state.mem); return as_nix_value_ptr(a.value);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -474,8 +461,9 @@ nix_value * nix_get_attr_byidx_lazy(
} }
const nix::Attr & a = (*v.attrs())[i]; const nix::Attr & a = (*v.attrs())[i];
*name = state->state.symbols[a.name].c_str(); *name = state->state.symbols[a.name].c_str();
nix_gc_incref(nullptr, a.value);
// Note: intentionally NOT calling forceValue() to keep the attribute lazy // Note: intentionally NOT calling forceValue() to keep the attribute lazy
return new_nix_value(a.value, state->state.mem); return as_nix_value_ptr(a.value);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -515,7 +503,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
auto & v = check_value_out(value); auto & v = check_value_out(value);
v.mkString(std::string_view(str), *value->mem); v.mkString(std::string_view(str));
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -526,7 +514,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
auto & v = check_value_out(value); auto & v = check_value_out(value);
v.mkPath(s->state.rootPath(nix::CanonPath(str)), s->state.mem); v.mkPath(s->state.rootPath(nix::CanonPath(str)));
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
+1 -1
View File
@@ -73,7 +73,7 @@ TEST_F(JSONValueTest, StringQuotes)
TEST_F(JSONValueTest, DISABLED_Path) TEST_F(JSONValueTest, DISABLED_Path)
{ {
Value v; Value v;
v.mkPath(state.rootPath(CanonPath("/test")), state.mem); v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
} }
} /* namespace nix */ } /* namespace nix */
+2 -2
View File
@@ -268,7 +268,7 @@ struct StringPrintingTests : LibExprTest
void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args) void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
{ {
Value v; Value v;
v.mkString(literal, state.mem); v.mkString(literal);
std::stringstream out; std::stringstream out;
printValue(state, out, v, PrintOptions{.maxStringLength = maxLength}); printValue(state, out, v, PrintOptions{.maxStringLength = maxLength});
@@ -353,7 +353,7 @@ TEST_F(ValuePrintingTests, ansiColorsStringElided)
TEST_F(ValuePrintingTests, ansiColorsPath) TEST_F(ValuePrintingTests, ansiColorsPath)
{ {
Value v; Value v;
v.mkPath(state.rootPath(CanonPath("puppy")), state.mem); v.mkPath(state.rootPath(CanonPath("puppy")));
test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true}); test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true});
} }
+30 -39
View File
@@ -81,20 +81,20 @@ static const char * makeImmutableString(std::string_view s)
return t; return t;
} }
StringData & StringData::alloc(EvalMemory & mem, size_t size) StringData & StringData::alloc(size_t size)
{ {
void * t = mem.allocBytes(sizeof(StringData) + size + 1); void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
if (!t) if (!t)
throw std::bad_alloc(); throw std::bad_alloc();
auto res = new (t) StringData(size); auto res = new (t) StringData(size);
return *res; return *res;
} }
const StringData & StringData::make(EvalMemory & mem, std::string_view s) const StringData & StringData::make(std::string_view s)
{ {
if (s.empty()) if (s.empty())
return ""_sds; return ""_sds;
auto & res = alloc(mem, s.size()); auto & res = alloc(s.size());
std::memcpy(&res.data_, s.data(), s.size()); std::memcpy(&res.data_, s.data(), s.size());
res.data_[s.size()] = '\0'; res.data_[s.size()] = '\0';
return res; return res;
@@ -205,7 +205,7 @@ bool Value::isTrivial() const
{ {
return !isa<tApp, tPrimOpApp>() return !isa<tApp, tPrimOpApp>()
&& (!isa<tThunk>() && (!isa<tThunk>()
|| (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs->empty()) || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr)); || dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr));
} }
@@ -849,9 +849,9 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
evalState.runDebugRepl(nullptr, trace.env, trace.expr); evalState.runDebugRepl(nullptr, trace.env, trace.expr);
} }
void Value::mkString(std::string_view s, EvalMemory & mem) void Value::mkString(std::string_view s)
{ {
mkStringNoCopy(StringData::make(mem, s)); mkStringNoCopy(StringData::make(s));
} }
Value::StringWithContext::Context * Value::StringWithContext::Context *
@@ -862,13 +862,13 @@ Value::StringWithContext::Context::fromBuilder(const NixStringContext & context,
auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size()); auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
std::ranges::transform( std::ranges::transform(
context, ctx->elems, [&](const NixStringContextElem & elt) { return &StringData::make(mem, elt.to_string()); }); context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
return ctx; return ctx;
} }
void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem) void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
{ {
mkStringNoCopy(StringData::make(mem, s), Value::StringWithContext::Context::fromBuilder(context, mem)); mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
} }
void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem) void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
@@ -876,9 +876,9 @@ void Value::mkStringMove(const StringData & s, const NixStringContext & context,
mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem)); mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
} }
void Value::mkPath(const SourcePath & path, EvalMemory & mem) void Value::mkPath(const SourcePath & path)
{ {
mkPath(&*path.accessor, StringData::make(mem, path.path.abs())); mkPath(&*path.accessor, StringData::make(path.path.abs()));
} }
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -943,7 +943,7 @@ void EvalState::mkPos(Value & v, PosIdx p)
auto origin = positions.originOf(p); auto origin = positions.originOf(p);
if (auto path = std::get_if<SourcePath>(&origin)) { if (auto path = std::get_if<SourcePath>(&origin)) {
auto attrs = buildBindings(3); auto attrs = buildBindings(3);
attrs.alloc(s.file).mkString(path->path.abs(), mem); attrs.alloc(s.file).mkString(path->path.abs());
makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column)); makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column));
v.mkAttrs(attrs); v.mkAttrs(attrs);
} else } else
@@ -1226,26 +1226,26 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
void ExprAttrs::eval(EvalState & state, Env & env, Value & v) void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
{ {
auto bindings = state.buildBindings(attrs->size() + dynamicAttrs->size()); auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
auto dynamicEnv = &env; auto dynamicEnv = &env;
bool sort = false; bool sort = false;
if (recursive) { if (recursive) {
/* Create a new environment that contains the attributes in /* Create a new environment that contains the attributes in
this `rec'. */ this `rec'. */
Env & env2(state.mem.allocEnv(attrs->size())); Env & env2(state.mem.allocEnv(attrs.size()));
env2.up = &env; env2.up = &env;
dynamicEnv = &env2; dynamicEnv = &env2;
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
AttrDefs::iterator overrides = attrs->find(state.s.overrides); AttrDefs::iterator overrides = attrs.find(state.s.overrides);
bool hasOverrides = overrides != attrs->end(); bool hasOverrides = overrides != attrs.end();
/* The recursive attributes are evaluated in the new /* The recursive attributes are evaluated in the new
environment, while the inherited attributes are evaluated environment, while the inherited attributes are evaluated
in the original environment. */ in the original environment. */
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs) { for (auto & i : attrs) {
Value * vAttr; Value * vAttr;
if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) { if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
vAttr = state.allocValue(); vAttr = state.allocValue();
@@ -1272,8 +1272,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
"while evaluating the `__overrides` attribute"); "while evaluating the `__overrides` attribute");
bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size())); bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size()));
for (auto & i : *vOverrides->attrs()) { for (auto & i : *vOverrides->attrs()) {
AttrDefs::iterator j = attrs->find(i.name); AttrDefs::iterator j = attrs.find(i.name);
if (j != attrs->end()) { if (j != attrs.end()) {
(*bindings.bindings)[j->second.displ] = i; (*bindings.bindings)[j->second.displ] = i;
env2.values[j->second.displ] = i.value; env2.values[j->second.displ] = i.value;
} else } else
@@ -1285,13 +1285,13 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
else { else {
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
for (auto & i : *attrs) for (auto & i : attrs)
bindings.insert( bindings.insert(
i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos); i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos);
} }
/* Dynamic attrs apply *after* rec and __overrides. */ /* Dynamic attrs apply *after* rec and __overrides. */
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
Value nameVal; Value nameVal;
i.nameExpr->eval(state, *dynamicEnv, nameVal); i.nameExpr->eval(state, *dynamicEnv, nameVal);
state.forceValue(nameVal, i.pos); state.forceValue(nameVal, i.pos);
@@ -1325,7 +1325,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
{ {
/* Create a new environment that contains the attributes in this /* Create a new environment that contains the attributes in this
`let'. */ `let'. */
Env & env2(state.mem.allocEnv(attrs->attrs->size())); Env & env2(state.mem.allocEnv(attrs->attrs.size()));
env2.up = &env; env2.up = &env;
Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr; Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1334,7 +1334,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
while the inherited attributes are evaluated in the original while the inherited attributes are evaluated in the original
environment. */ environment. */
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs->attrs) { for (auto & i : attrs->attrs) {
env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
} }
@@ -1751,9 +1751,9 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
// 4: about 60 // 4: about 60
// 5: under 10 // 5: under 10
// This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
SmallValueVector<4> vArgs(args->size()); SmallValueVector<4> vArgs(args.size());
for (size_t i = 0; i < args->size(); ++i) for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = (*args)[i]->maybeThunk(state, env); vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, vArgs, v, pos); state.callFunction(vFun, vArgs, v, pos);
} }
@@ -2139,9 +2139,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
for (const auto & part : strings) { for (const auto & part : strings) {
resultStr += *part; resultStr += *part;
} }
v.mkPath(state.rootPath(CanonPath(resultStr)), state.mem); v.mkPath(state.rootPath(CanonPath(resultStr)));
} else { } else {
auto & resultStr = StringData::alloc(state.mem, sSize); auto & resultStr = StringData::alloc(sSize);
auto * tmp = resultStr.data(); auto * tmp = resultStr.data();
for (const auto & part : strings) { for (const auto & part : strings) {
std::memcpy(tmp, part->data(), part->size()); std::memcpy(tmp, part->data(), part->size());
@@ -2188,8 +2188,6 @@ void EvalState::forceValueDeep(Value & v)
std::set<const Value *> seen; std::set<const Value *> seen;
[&, &state(*this)](this const auto & recurse, Value & v) { [&, &state(*this)](this const auto & recurse, Value & v) {
auto _level = state.addCallDepth(v.determinePos(noPos));
if (!seen.insert(&v).second) if (!seen.insert(&v).second)
return; return;
@@ -2216,15 +2214,8 @@ void EvalState::forceValueDeep(Value & v)
} }
else if (v.isList()) { else if (v.isList()) {
size_t index = 0;
for (auto v2 : v.listView()) for (auto v2 : v.listView())
try { recurse(*v2);
recurse(*v2);
index++;
} catch (Error & e) {
state.addErrorTrace(e, "while evaluating list element at index %1%", index);
throw;
}
} }
}(v); }(v);
} }
@@ -3199,7 +3190,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
if (EvalSettings::isPseudoUrl(value)) { if (EvalSettings::isPseudoUrl(value)) {
try { try {
auto accessor = fetchers::downloadTarball(*store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); auto accessor = fetchers::downloadTarball(store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
return finish(this->storePath(storePath)); return finish(this->storePath(storePath));
} catch (Error & e) { } catch (Error & e) {
+12 -26
View File
@@ -395,13 +395,9 @@ struct ExprAttrs : Expr
} }
}; };
typedef std::pmr::map<Symbol, AttrDef> AttrDefs; typedef std::map<Symbol, AttrDef> AttrDefs;
/** AttrDefs attrs;
* attrs will never be null. we use std::optional so that we can call emplace() to re-initialize the value with a std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
* new pmr::map using a different allocator (move assignment will copy into the old allocator)
*/
std::optional<AttrDefs> attrs;
std::unique_ptr<std::pmr::vector<Expr *>> inheritFromExprs;
struct DynamicAttrDef struct DynamicAttrDef
{ {
@@ -413,20 +409,13 @@ struct ExprAttrs : Expr
, pos(pos) {}; , pos(pos) {};
}; };
typedef std::pmr::vector<DynamicAttrDef> DynamicAttrDefs; typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
/** DynamicAttrDefs dynamicAttrs;
* dynamicAttrs will never be null. See comment on AttrDefs above.
*/
std::optional<DynamicAttrDefs> dynamicAttrs;
ExprAttrs(const PosIdx & pos) ExprAttrs(const PosIdx & pos)
: recursive(false) : recursive(false)
, pos(pos) , pos(pos) {};
, attrs(AttrDefs{})
, dynamicAttrs(DynamicAttrDefs{}) {};
ExprAttrs() ExprAttrs()
: recursive(false) : recursive(false) {};
, attrs(AttrDefs{})
, dynamicAttrs(DynamicAttrDefs{}) {};
PosIdx getPos() const override PosIdx getPos() const override
{ {
@@ -592,14 +581,11 @@ public:
struct ExprCall : Expr struct ExprCall : Expr
{ {
Expr * fun; Expr * fun;
/** std::vector<Expr *> args;
* args will never be null. See comment on ExprAttrs::AttrDefs below.
*/
std::optional<std::pmr::vector<Expr *>> args;
PosIdx pos; PosIdx pos;
std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118 std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118
ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args) ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
: fun(fun) : fun(fun)
, args(args) , args(args)
, pos(pos) , pos(pos)
@@ -607,7 +593,7 @@ struct ExprCall : Expr
{ {
} }
ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos) ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
: fun(fun) : fun(fun)
, args(args) , args(args)
, pos(pos) , pos(pos)
@@ -839,7 +825,7 @@ public:
// we define some calls to add explicitly so that the argument can be passed in as initializer lists // we define some calls to add explicitly so that the argument can be passed in as initializer lists
template<class C> template<class C>
[[gnu::always_inline]] [[gnu::always_inline]]
C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args) C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
requires(std::same_as<C, ExprCall>) requires(std::same_as<C, ExprCall>)
{ {
return alloc.new_object<C>(pos, fun, std::move(args)); return alloc.new_object<C>(pos, fun, std::move(args));
@@ -847,7 +833,7 @@ public:
template<class C> template<class C>
[[gnu::always_inline]] [[gnu::always_inline]]
C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos) C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
requires(std::same_as<C, ExprCall>) requires(std::same_as<C, ExprCall>)
{ {
return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos)); return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));
+16 -16
View File
@@ -126,8 +126,8 @@ inline void ParserState::addAttr(
for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
ExprAttrs * nested; ExprAttrs * nested;
if (i->symbol) { if (i->symbol) {
ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(i->symbol); ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
if (j != attrs->attrs->end()) { if (j != attrs->attrs.end()) {
nested = dynamic_cast<ExprAttrs *>(j->second.e); nested = dynamic_cast<ExprAttrs *>(j->second.e);
if (!nested) { if (!nested) {
attrPath.erase(i + 1, attrPath.end()); attrPath.erase(i + 1, attrPath.end());
@@ -135,11 +135,11 @@ inline void ParserState::addAttr(
} }
} else { } else {
nested = exprs.add<ExprAttrs>(); nested = exprs.add<ExprAttrs>();
(*attrs->attrs)[i->symbol] = ExprAttrs::AttrDef(nested, pos); attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
} }
} else { } else {
nested = exprs.add<ExprAttrs>(); nested = exprs.add<ExprAttrs>();
attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
} }
attrs = nested; attrs = nested;
} }
@@ -148,7 +148,7 @@ inline void ParserState::addAttr(
if (i->symbol) { if (i->symbol) {
addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos)); addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
} else { } else {
attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
} }
auto it = lexerState.positionToDocComment.find(pos); auto it = lexerState.positionToDocComment.find(pos);
@@ -165,8 +165,8 @@ inline void ParserState::addAttr(
inline void inline void
ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def) ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
{ {
ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(symbol); ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
if (j != attrs->attrs->end()) { if (j != attrs->attrs.end()) {
// This attr path is already defined. However, if both // This attr path is already defined. However, if both
// e and the expr pointed by the attr path are two attribute sets, // e and the expr pointed by the attr path are two attribute sets,
// we want to merge them. // we want to merge them.
@@ -181,8 +181,8 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
// See https://github.com/NixOS/nix/issues/9020. // See https://github.com/NixOS/nix/issues/9020.
if (jAttrs && ae) { if (jAttrs && ae) {
if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
jAttrs->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(); jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
for (auto & ad : *ae->attrs) { for (auto & ad : ae->attrs) {
if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e); auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e); auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
@@ -192,12 +192,12 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
addAttr(jAttrs, attrPath, ad.first, std::move(ad.second)); addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
attrPath.pop_back(); attrPath.pop_back();
} }
ae->attrs->clear(); ae->attrs.clear();
jAttrs->dynamicAttrs->insert( jAttrs->dynamicAttrs.insert(
jAttrs->dynamicAttrs->end(), jAttrs->dynamicAttrs.end(),
std::make_move_iterator(ae->dynamicAttrs->begin()), std::make_move_iterator(ae->dynamicAttrs.begin()),
std::make_move_iterator(ae->dynamicAttrs->end())); std::make_move_iterator(ae->dynamicAttrs.end()));
ae->dynamicAttrs->clear(); ae->dynamicAttrs.clear();
if (ae->inheritFromExprs) { if (ae->inheritFromExprs) {
jAttrs->inheritFromExprs->insert( jAttrs->inheritFromExprs->insert(
jAttrs->inheritFromExprs->end(), jAttrs->inheritFromExprs->end(),
@@ -210,7 +210,7 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
} }
} else { } else {
// This attr path is not defined. Let's create it. // This attr path is not defined. Let's create it.
attrs->attrs->emplace(symbol, def); attrs->attrs.emplace(symbol, def);
def.e->setName(symbol); def.e->setName(symbol);
} }
} }
+4 -4
View File
@@ -232,13 +232,13 @@ public:
* Allocate StringData on the (possibly) GC-managed heap and copy * Allocate StringData on the (possibly) GC-managed heap and copy
* the contents of s to it. * the contents of s to it.
*/ */
static const StringData & make(EvalMemory & mem, std::string_view s); static const StringData & make(std::string_view s);
/** /**
* Allocate StringData on the (possibly) GC-managed heap. * Allocate StringData on the (possibly) GC-managed heap.
* @param size Length of the string (without the NUL terminator). * @param size Length of the string (without the NUL terminator).
*/ */
static StringData & alloc(EvalMemory & mem, size_t size); static StringData & alloc(size_t size);
size_t size() const size_t size() const
{ {
@@ -1147,13 +1147,13 @@ public:
setStorage(StringWithContext{.str = &s, .context = context}); setStorage(StringWithContext{.str = &s, .context = context});
} }
void mkString(std::string_view s, EvalMemory & mem); void mkString(std::string_view s);
void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem); void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem); void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
void mkPath(const SourcePath & path, EvalMemory & mem); void mkPath(const SourcePath & path);
inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
{ {
+1 -1
View File
@@ -151,7 +151,7 @@ public:
bool string(string_t & val) override bool string(string_t & val) override
{ {
forceNoNullByte(val); forceNoNullByte(val);
rs->value(state).mkString(val, state.mem); rs->value(state).mkString(val);
rs->add(); rs->add();
return true; return true;
} }
+14 -31
View File
@@ -74,9 +74,9 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
{ {
typedef const AttrDefs::value_type * Attr; typedef const decltype(attrs)::value_type * Attr;
std::vector<Attr> sorted; std::vector<Attr> sorted;
for (auto & i : *attrs) for (auto & i : attrs)
sorted.push_back(&i); sorted.push_back(&i);
std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) { std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
std::string_view sa = symbols[a->first], sb = symbols[b->first]; std::string_view sa = symbols[a->first], sb = symbols[b->first];
@@ -122,7 +122,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
str << "; "; str << "; ";
} }
} }
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
str << "\"${"; str << "\"${";
i.nameExpr->show(symbols, str); i.nameExpr->show(symbols, str);
str << "}\" = "; str << "}\" = ";
@@ -191,7 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
{ {
str << '('; str << '(';
fun->show(symbols, str); fun->show(symbols, str);
for (auto e : *args) { for (auto e : args) {
str << ' '; str << ' ';
e->show(symbols, str); e->show(symbols, str);
} }
@@ -401,26 +401,15 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
AttrDefs newAttrs{std::move(*attrs), arena};
attrs.emplace(std::move(newAttrs), arena);
DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
if (inheritFromExprs)
inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
}
if (es.debugRepl) if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env)); es.exprEnvs.insert(std::make_pair(this, env));
if (recursive) { if (recursive) {
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> { auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->size()); auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs.size());
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs) for (auto & i : attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++); newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv; return newEnv;
}(); }();
@@ -428,20 +417,20 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
// No need to sort newEnv since attrs is in sorted order. // No need to sort newEnv since attrs is in sorted order.
auto inheritFromEnv = bindInheritSources(es, newEnv); auto inheritFromEnv = bindInheritSources(es, newEnv);
for (auto & i : *attrs) for (auto & i : attrs)
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, newEnv); i.nameExpr->bindVars(es, newEnv);
i.valueExpr->bindVars(es, newEnv); i.valueExpr->bindVars(es, newEnv);
} }
} else { } else {
auto inheritFromEnv = bindInheritSources(es, env); auto inheritFromEnv = bindInheritSources(es, env);
for (auto & i : *attrs) for (auto & i : attrs)
i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, env); i.nameExpr->bindVars(es, env);
i.valueExpr->bindVars(es, env); i.valueExpr->bindVars(es, env);
} }
@@ -486,27 +475,21 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
std::pmr::vector<Expr *> newArgs{std::move(*args), arena};
args.emplace(std::move(newArgs), arena);
}
if (es.debugRepl) if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env)); es.exprEnvs.insert(std::make_pair(this, env));
fun->bindVars(es, env); fun->bindVars(es, env);
for (auto e : *args) for (auto e : args)
e->bindVars(es, env); e->bindVars(es, env);
} }
void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> { auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size()); auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs.size());
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs->attrs) for (auto & i : attrs->attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++); newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv; return newEnv;
}(); }();
@@ -514,7 +497,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
// No need to sort newEnv since attrs->attrs is in sorted order. // No need to sort newEnv since attrs->attrs is in sorted order.
auto inheritFromEnv = attrs->bindInheritSources(es, newEnv); auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
for (auto & i : *attrs->attrs) for (auto & i : attrs->attrs)
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
if (es.debugRepl) if (es.debugRepl)
+10 -10
View File
@@ -115,7 +115,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) { static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
if (auto e2 = dynamic_cast<ExprCall *>(fn)) { if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
e2->args->push_back(arg); e2->args.push_back(arg);
return fn; return fn;
} }
return exprs.add<ExprCall>(pos, fn, {arg}); return exprs.add<ExprCall>(pos, fn, {arg});
@@ -129,7 +129,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%type <Expr *> start expr expr_function expr_if expr_op %type <Expr *> start expr expr_function expr_if expr_op
%type <Expr *> expr_select expr_simple expr_app %type <Expr *> expr_select expr_simple expr_app
%type <Expr *> expr_pipe_from expr_pipe_into %type <Expr *> expr_pipe_from expr_pipe_into
%type <std::pmr::vector<Expr *>> list %type <std::vector<Expr *>> list
%type <ExprAttrs *> binds binds1 %type <ExprAttrs *> binds binds1
%type <FormalsBuilder> formals formal_set %type <FormalsBuilder> formals formal_set
%type <Formal> formal %type <Formal> formal
@@ -211,7 +211,7 @@ expr_function
| WITH expr ';' expr_function | WITH expr ';' expr_function
{ $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); } { $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); }
| LET binds IN_KW expr_function | LET binds IN_KW expr_function
{ if (!$2->dynamicAttrs->empty()) { if (!$2->dynamicAttrs.empty())
throw ParseError({ throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in let"), .msg = HintFmt("dynamic attributes not allowed in let"),
.pos = state->positions[CUR_POS] .pos = state->positions[CUR_POS]
@@ -413,9 +413,9 @@ binds1
| binds[accum] INHERIT attrs ';' | binds[accum] INHERIT attrs ';'
{ $$ = $accum; { $$ = $accum;
for (auto & [i, iPos] : $attrs) { for (auto & [i, iPos] : $attrs) {
if ($accum->attrs->find(i.symbol) != $accum->attrs->end()) if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos); state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs->emplace( $accum->attrs.emplace(
i.symbol, i.symbol,
ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
} }
@@ -423,13 +423,13 @@ binds1
| binds[accum] INHERIT '(' expr ')' attrs ';' | binds[accum] INHERIT '(' expr ')' attrs ';'
{ $$ = $accum; { $$ = $accum;
if (!$accum->inheritFromExprs) if (!$accum->inheritFromExprs)
$accum->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(); $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$accum->inheritFromExprs->push_back($expr); $accum->inheritFromExprs->push_back($expr);
auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1); auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : $attrs) { for (auto & [i, iPos] : $attrs) {
if ($accum->attrs->find(i.symbol) != $accum->attrs->end()) if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos); state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs->emplace( $accum->attrs.emplace(
i.symbol, i.symbol,
ExprAttrs::AttrDef( ExprAttrs::AttrDef(
state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol), state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol),
+22 -24
View File
@@ -53,7 +53,7 @@ RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
static inline Value * mkString(EvalState & state, const std::csub_match & match) static inline Value * mkString(EvalState & state, const std::csub_match & match)
{ {
Value * v = state.allocValue(); Value * v = state.allocValue();
v->mkString({match.first, match.second}, state.mem); v->mkString({match.first, match.second});
return v; return v;
} }
@@ -230,12 +230,12 @@ void derivationToValue(
NixStringContextElem::DrvDeep{.drvPath = storePath}, NixStringContextElem::DrvDeep{.drvPath = storePath},
}, },
state.mem); state.mem);
attrs.alloc(state.s.name).mkString(drv.env["name"], state.mem); attrs.alloc(state.s.name).mkString(drv.env["name"]);
auto list = state.buildList(drv.outputs.size()); auto list = state.buildList(drv.outputs.size());
for (const auto & [i, o] : enumerate(drv.outputs)) { for (const auto & [i, o] : enumerate(drv.outputs)) {
mkOutputString(state, attrs, storePath, o); mkOutputString(state, attrs, storePath, o);
(list[i] = state.allocValue())->mkString(o.first, state.mem); (list[i] = state.allocValue())->mkString(o.first);
} }
attrs.alloc(state.s.outputs).mkList(list); attrs.alloc(state.s.outputs).mkList(list);
@@ -519,7 +519,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
v.mkStringNoCopy("lambda"_sds); v.mkStringNoCopy("lambda"_sds);
break; break;
case nExternal: case nExternal:
v.mkString(args[0]->external()->typeOf(), state.mem); v.mkString(args[0]->external()->typeOf());
break; break;
case nFloat: case nFloat:
v.mkStringNoCopy("float"_sds); v.mkStringNoCopy("float"_sds);
@@ -1176,7 +1176,7 @@ static void prim_getEnv(EvalState & state, const PosIdx pos, Value ** args, Valu
{ {
std::string name( std::string name(
state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""), state.mem); v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
} }
static RegisterPrimOp primop_getEnv({ static RegisterPrimOp primop_getEnv({
@@ -1842,10 +1842,8 @@ static RegisterPrimOp primop_derivationStrict(
‘out’. */ ‘out’. */
static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v) static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v)
{ {
v.mkString( v.mkString(hashPlaceholder(
hashPlaceholder(state.forceStringNoCtx( state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")));
*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")),
state.mem);
} }
static RegisterPrimOp primop_placeholder({ static RegisterPrimOp primop_placeholder({
@@ -2029,7 +2027,7 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
state.forceValue(*args[0], pos); state.forceValue(*args[0], pos);
if (args[0]->type() == nPath) { if (args[0]->type() == nPath) {
auto path = args[0]->path(); auto path = args[0]->path();
v.mkPath(path.path.isRoot() ? path : path.parent(), state.mem); v.mkPath(path.path.isRoot() ? path : path.parent());
} else { } else {
NixStringContext context; NixStringContext context;
auto path = state.coerceToString( auto path = state.coerceToString(
@@ -2146,7 +2144,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
auto path = auto path =
state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
v.mkPath(state.findFile(lookupPath, path, pos), state.mem); v.mkPath(state.findFile(lookupPath, path, pos));
} }
static RegisterPrimOp primop_findFile( static RegisterPrimOp primop_findFile(
@@ -2295,7 +2293,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
auto path = realisePath(state, pos, *args[1]); auto path = realisePath(state, pos, *args[1]);
v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem); v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
} }
static RegisterPrimOp primop_hashFile({ static RegisterPrimOp primop_hashFile({
@@ -2384,7 +2382,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val
// detailed node info quickly in this case we produce a thunk to // detailed node info quickly in this case we produce a thunk to
// query the file type lazily. // query the file type lazily.
auto epath = state.allocValue(); auto epath = state.allocValue();
epath->mkPath(path / name, state.mem); epath->mkPath(path / name);
if (!readFileType) if (!readFileType)
readFileType = &state.getBuiltin("readFileType"); readFileType = &state.getBuiltin("readFileType");
attr.mkApp(readFileType, epath); attr.mkApp(readFileType, epath);
@@ -2765,7 +2763,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId
/* Call the filter function. The first argument is the path, the /* Call the filter function. The first argument is the path, the
second is a string indicating the type of the file. */ second is a string indicating the type of the file. */
Value arg1; Value arg1;
arg1.mkString(path.path.abs(), mem); arg1.mkString(path.path.abs());
// assert that type is not "unknown" // assert that type is not "unknown"
Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))}; Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))};
@@ -4543,7 +4541,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value ** args,
auto s = auto s =
state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false), state.mem); v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
} }
static RegisterPrimOp primop_hashString({ static RegisterPrimOp primop_hashString({
@@ -4576,7 +4574,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value ** args,
HashFormat hf = parseHashFormat( HashFormat hf = parseHashFormat(
state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI), state.mem); v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
} }
static RegisterPrimOp primop_convertHash({ static RegisterPrimOp primop_convertHash({
@@ -4994,8 +4992,8 @@ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value ** args
state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName"); state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName");
DrvName parsed(name); DrvName parsed(name);
auto attrs = state.buildBindings(2); auto attrs = state.buildBindings(2);
attrs.alloc(state.s.name).mkString(parsed.name, state.mem); attrs.alloc(state.s.name).mkString(parsed.name);
attrs.alloc("version").mkString(parsed.version, state.mem); attrs.alloc("version").mkString(parsed.version);
v.mkAttrs(attrs); v.mkAttrs(attrs);
} }
@@ -5050,7 +5048,7 @@ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value ** args
} }
auto list = state.buildList(components.size()); auto list = state.buildList(components.size());
for (const auto & [n, component] : enumerate(components)) for (const auto & [n, component] : enumerate(components))
(list[n] = state.allocValue())->mkString(std::move(component), state.mem); (list[n] = state.allocValue())->mkString(std::move(component));
v.mkList(list); v.mkList(list);
} }
@@ -5194,7 +5192,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
}); });
if (!settings.pureEval) if (!settings.pureEval)
v.mkString(settings.getCurrentSystem(), mem); v.mkString(settings.getCurrentSystem());
addConstant( addConstant(
"__currentSystem", "__currentSystem",
v, v,
@@ -5226,7 +5224,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
.impureOnly = true, .impureOnly = true,
}); });
v.mkString(nixVersion, mem); v.mkString(nixVersion);
addConstant( addConstant(
"__nixVersion", "__nixVersion",
v, v,
@@ -5251,7 +5249,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
)", )",
}); });
v.mkString(store->storeDir, mem); v.mkString(store->storeDir);
addConstant( addConstant(
"__storeDir", "__storeDir",
v, v,
@@ -5316,8 +5314,8 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
auto list = buildList(lookupPath.elements.size()); auto list = buildList(lookupPath.elements.size());
for (const auto & [n, i] : enumerate(lookupPath.elements)) { for (const auto & [n, i] : enumerate(lookupPath.elements)) {
auto attrs = buildBindings(2); auto attrs = buildBindings(2);
attrs.alloc("path").mkString(i.path.s, mem); attrs.alloc("path").mkString(i.path.s);
attrs.alloc("prefix").mkString(i.prefix.s, mem); attrs.alloc("prefix").mkString(i.prefix.s);
(list[n] = allocValue())->mkAttrs(attrs); (list[n] = allocValue())->mkAttrs(attrs);
} }
v.mkList(list); v.mkList(list);
+2 -2
View File
@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
NixStringContext context; NixStringContext context;
auto s = state.coerceToString( auto s = state.coerceToString(
pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
v.mkString(*s, state.mem); v.mkString(*s);
} }
static RegisterPrimOp primop_unsafeDiscardStringContext({ static RegisterPrimOp primop_unsafeDiscardStringContext({
@@ -218,7 +218,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
if (!info.second.outputs.empty()) { if (!info.second.outputs.empty()) {
auto list = state.buildList(info.second.outputs.size()); auto list = state.buildList(info.second.outputs.size());
for (const auto & [i, output] : enumerate(info.second.outputs)) for (const auto & [i, output] : enumerate(info.second.outputs))
(list[i] = state.allocValue())->mkString(output, state.mem); (list[i] = state.allocValue())->mkString(output);
infoAttrs.alloc(state.s.outputs).mkList(list); infoAttrs.alloc(state.s.outputs).mkList(list);
} }
attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
+4 -4
View File
@@ -81,17 +81,17 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
attrs.insert_or_assign("rev", rev->gitRev()); attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store); auto [storePath, input2] = input.fetchToStore(state.fetchSettings, state.store);
auto attrs2 = state.buildBindings(8); auto attrs2 = state.buildBindings(8);
state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath)); state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
if (input2.getRef()) if (input2.getRef())
attrs2.alloc("branch").mkString(*input2.getRef(), state.mem); attrs2.alloc("branch").mkString(*input2.getRef());
// Backward compatibility: set 'rev' to // Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree. // 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1)); auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
attrs2.alloc("rev").mkString(rev2.gitRev(), state.mem); attrs2.alloc("rev").mkString(rev2.gitRev());
attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12), state.mem); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
if (auto revCount = input2.getRevCount()) if (auto revCount = input2.getRevCount())
attrs2.alloc("revCount").mkInt(*revCount); attrs2.alloc("revCount").mkInt(*revCount);
v.mkAttrs(attrs2); v.mkAttrs(attrs2);
+17 -13
View File
@@ -35,7 +35,7 @@ void emitTreeAttrs(
// FIXME: support arbitrary input attributes. // FIXME: support arbitrary input attributes.
if (auto narHash = input.getNarHash()) if (auto narHash = input.getNarHash())
attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem); attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (input.getType() == "git") if (input.getType() == "git")
attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
@@ -43,13 +43,13 @@ void emitTreeAttrs(
if (!forceDirty) { if (!forceDirty) {
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
attrs.alloc("rev").mkString(rev->gitRev(), state.mem); attrs.alloc("rev").mkString(rev->gitRev());
attrs.alloc("shortRev").mkString(rev->gitShortRev(), state.mem); attrs.alloc("shortRev").mkString(rev->gitShortRev());
} else if (emptyRevFallback) { } else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(HashAlgorithm::SHA1); auto emptyHash = Hash(HashAlgorithm::SHA1);
attrs.alloc("rev").mkString(emptyHash.gitRev(), state.mem); attrs.alloc("rev").mkString(emptyHash.gitRev());
attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
} }
if (auto revCount = input.getRevCount()) if (auto revCount = input.getRevCount())
@@ -59,14 +59,13 @@ void emitTreeAttrs(
} }
if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) { if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
attrs.alloc("dirtyRev").mkString(*dirtyRev, state.mem); attrs.alloc("dirtyRev").mkString(*dirtyRev);
attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"), state.mem); attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
} }
if (auto lastModified = input.getLastModified()) { if (auto lastModified = input.getLastModified()) {
attrs.alloc("lastModified").mkInt(*lastModified); attrs.alloc("lastModified").mkInt(*lastModified);
attrs.alloc("lastModifiedDate") attrs.alloc("lastModifiedDate").mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
.mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")), state.mem);
} }
v.mkAttrs(attrs); v.mkAttrs(attrs);
@@ -151,6 +150,11 @@ static void fetchTree(
attrs.emplace("exportIgnore", Explicit<bool>{true}); attrs.emplace("exportIgnore", Explicit<bool>{true});
} }
// fetchTree should fetch git repos with shallow = true by default
if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
attrs.emplace("shallow", Explicit<bool>{true});
}
if (!params.allowNameArgument) if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher) state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher)
@@ -190,7 +194,7 @@ static void fetchTree(
} }
if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first; input = lookupInRegistries(state.fetchSettings, state.store, input, fetchers::UseRegistries::Limited).first;
if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) { if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) {
if (input.getNarHash()) if (input.getNarHash())
@@ -216,7 +220,7 @@ static void fetchTree(
} }
auto cachedInput = auto cachedInput =
state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No); state.inputCache->getAccessor(state.fetchSettings, state.store, input, fetchers::UseRegistries::No);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
@@ -476,10 +480,10 @@ static void fetch(
auto storePath = unpack ? fetchToStore( auto storePath = unpack ? fetchToStore(
state.fetchSettings, state.fetchSettings,
*state.store, *state.store,
fetchers::downloadTarball(*state.store, state.fetchSettings, *url), fetchers::downloadTarball(state.store, state.fetchSettings, *url),
FetchMode::Copy, FetchMode::Copy,
name) name)
: fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
if (expectedHash) { if (expectedHash) {
auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash
+2 -2
View File
@@ -126,7 +126,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
case toml::value_t::string: { case toml::value_t::string: {
auto s = toml::get<std::string_view>(t); auto s = toml::get<std::string_view>(t);
forceNoNullByte(s); forceNoNullByte(s);
v.mkString(s, state.mem); v.mkString(s);
} break; } break;
case toml::value_t::local_datetime: case toml::value_t::local_datetime:
case toml::value_t::offset_datetime: case toml::value_t::offset_datetime:
@@ -142,7 +142,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
s << t; s << t;
auto str = s.view(); auto str = s.view();
forceNoNullByte(str); forceNoNullByte(str);
attrs.alloc("value").mkString(str, state.mem); attrs.alloc("value").mkString(str);
v.mkAttrs(attrs); v.mkAttrs(attrs);
} else { } else {
throw std::runtime_error("Dates and times are not supported"); throw std::runtime_error("Dates and times are not supported");
-2
View File
@@ -16,8 +16,6 @@ json printValueAsJSON(
{ {
checkInterrupt(); checkInterrupt();
auto _level = state.addCallDepth(pos);
if (strict) if (strict)
state.forceValue(v, pos); state.forceValue(v, pos);
+1 -1
View File
@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
auto result = repo->dereferenceSingletonDirectory(sink->flush()); auto result = repo->dereferenceSingletonDirectory(sink->flush());
auto accessor = repo->getAccessor(result, {}, getRepoName()); auto accessor = repo->getAccessor(result, false, getRepoName());
auto entries = accessor->readDirectory(CanonPath::root); auto entries = accessor->readDirectory(CanonPath::root);
ASSERT_EQ(entries.size(), 5u); ASSERT_EQ(entries.size(), 5u);
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
+1 -1
View File
@@ -196,7 +196,7 @@ TEST_F(GitTest, submodulePeriodSupport)
{"ref", "main"}, {"ref", "main"},
}); });
auto [accessor, i] = input.getAccessor(settings, *store); auto [accessor, i] = input.getAccessor(settings, store);
ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n");
} }
+29 -26
View File
@@ -6,7 +6,6 @@
#include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/url.hh" #include "nix/util/url.hh"
#include "nix/util/archive.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -111,7 +110,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
return std::move(*res); return std::move(*res);
} }
std::optional<std::string> Input::getFingerprint(Store & store) const std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{ {
if (!scheme) if (!scheme)
return std::nullopt; return std::nullopt;
@@ -190,7 +189,7 @@ bool Input::contains(const Input & other) const
} }
// FIXME: remove // FIXME: remove
std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store & store) const std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, ref<Store> store) const
{ {
if (!scheme) if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@@ -200,9 +199,9 @@ std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store
auto [accessor, result] = getAccessorUnchecked(settings, store); auto [accessor, result] = getAccessorUnchecked(settings, store);
auto storePath = auto storePath =
nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName()); nix::fetchToStore(settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto narHash = store.queryPathInfo(storePath)->narHash; auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
result.attrs.insert_or_assign("__final", Explicit<bool>(true)); result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -269,13 +268,27 @@ void Input::checkLocks(Input specified, Input & result)
} }
} }
if (auto prevLastModified = specified.getLastModified()) {
if (result.getLastModified() != prevLastModified)
throw Error(
"'lastModified' attribute mismatch in input '%s', expected %d, got %d",
result.to_string(),
*prevLastModified,
result.getLastModified().value_or(-1));
}
if (auto prevRev = specified.getRev()) { if (auto prevRev = specified.getRev()) {
if (result.getRev() != prevRev) if (result.getRev() != prevRev)
throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev());
} }
if (auto prevRevCount = specified.getRevCount()) {
if (result.getRevCount() != prevRevCount)
throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount);
}
} }
std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, ref<Store> store) const
{ {
try { try {
auto [accessor, result] = getAccessorUnchecked(settings, store); auto [accessor, result] = getAccessorUnchecked(settings, store);
@@ -291,7 +304,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settin
} }
} }
std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, Store & store) const std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, ref<Store> store) const
{ {
// FIXME: cache the accessor // FIXME: cache the accessor
@@ -311,13 +324,13 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
*/ */
if (isFinal() && getNarHash()) { if (isFinal() && getNarHash()) {
try { try {
auto storePath = computeStorePath(store); auto storePath = computeStorePath(*store);
store.ensurePath(storePath); store->ensurePath(storePath);
debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath)); debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
auto accessor = store.requireStoreObjectAccessor(storePath); auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->fingerprint = getFingerprint(store); accessor->fingerprint = getFingerprint(store);
@@ -327,7 +340,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
if (accessor->fingerprint) { if (accessor->fingerprint) {
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/"); auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings.getCache()->upsert(cacheKey, store, {}, storePath); settings.getCache()->upsert(cacheKey, *store, {}, storePath);
} }
accessor->setPathDisplay("«" + to_string() + "»"); accessor->setPathDisplay("«" + to_string() + "»");
@@ -355,10 +368,10 @@ Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash>
return scheme->applyOverrides(*this, ref, rev); return scheme->applyOverrides(*this, ref, rev);
} }
void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const void Input::clone(const Settings & settings, const Path & destDir) const
{ {
assert(scheme); assert(scheme);
scheme->clone(settings, store, *this, destDir); scheme->clone(settings, *this, destDir);
} }
std::optional<std::filesystem::path> Input::getSourcePath() const std::optional<std::filesystem::path> Input::getSourcePath() const
@@ -471,19 +484,9 @@ void InputScheme::putFile(
throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
} }
void InputScheme::clone( void InputScheme::clone(const Settings & settings, const Input & input, const Path & destDir) const
const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
{ {
if (std::filesystem::exists(destDir)) throw Error("do not know how to clone input '%s'", input.to_string());
throw Error("cannot clone into existing path %s", destDir);
auto [accessor, input2] = getAccessor(settings, store, input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
RestoreSink sink(/*startFsync=*/false);
sink.dstPath = destDir;
copyRecursive(*accessor, CanonPath::root, sink, CanonPath::root);
} }
std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
+1 -1
View File
@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
auto url = api.endpoint + "/objects/batch"; auto url = api.endpoint + "/objects/batch";
const auto & authHeader = api.authHeader; const auto & authHeader = api.authHeader;
FileTransferRequest request(parseURL(url)); FileTransferRequest request(parseURL(url));
request.method = HttpMethod::Post; request.method = HttpMethod::POST;
Headers headers; Headers headers;
if (authHeader.has_value()) if (authHeader.has_value())
headers.push_back({"Authorization", *authHeader}); headers.push_back({"Authorization", *authHeader});
+24 -24
View File
@@ -541,15 +541,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
} }
/** /**
* A 'GitSourceAccessor' with no regard for export-ignore. * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
*/ */
ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options); ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
ref<SourceAccessor> ref<SourceAccessor>
getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override; getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
ref<SourceAccessor> ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
ref<GitFileSystemObjectSink> getFileSystemObjectSink() override; ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
@@ -669,7 +668,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
{ {
auto accessor = getAccessor(treeHash, {}, ""); auto accessor = getAccessor(treeHash, false, "");
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
@@ -717,17 +716,15 @@ struct GitSourceAccessor : SourceAccessor
ref<GitRepoImpl> repo; ref<GitRepoImpl> repo;
Object root; Object root;
std::optional<lfs::Fetch> lfsFetch = std::nullopt; std::optional<lfs::Fetch> lfsFetch = std::nullopt;
GitAccessorOptions options;
}; };
Sync<State> state_; Sync<State> state_;
GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options) GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
: state_{State{ : state_{State{
.repo = repo_, .repo = repo_,
.root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()), .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
.lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt, .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
.options = options,
}} }}
{ {
} }
@@ -1257,26 +1254,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
} }
}; };
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options) ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
{ {
auto self = ref<GitRepoImpl>(shared_from_this()); auto self = ref<GitRepoImpl>(shared_from_this());
return make_ref<GitSourceAccessor>(self, rev, options); return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
} }
ref<SourceAccessor> ref<SourceAccessor>
GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
{ {
auto self = ref<GitRepoImpl>(shared_from_this()); auto self = ref<GitRepoImpl>(shared_from_this());
ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options); ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
rawGitAccessor->setPathDisplay(std::move(displayPrefix)); rawGitAccessor->setPathDisplay(std::move(displayPrefix));
if (options.exportIgnore) if (exportIgnore)
return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev); return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
else else
return rawGitAccessor; return rawGitAccessor;
} }
ref<SourceAccessor> GitRepoImpl::getAccessor( ref<SourceAccessor>
const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
{ {
auto self = ref<GitRepoImpl>(shared_from_this()); auto self = ref<GitRepoImpl>(shared_from_this());
ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create( ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1286,9 +1283,10 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(
boost::unordered_flat_set<CanonPath>{CanonPath::root}, boost::unordered_flat_set<CanonPath>{CanonPath::root},
std::move(makeNotAllowedError)) std::move(makeNotAllowedError))
.cast<SourceAccessor>(); .cast<SourceAccessor>();
if (options.exportIgnore) if (exportIgnore)
fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt); return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
return fileAccessor; else
return fileAccessor;
} }
ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink() ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
@@ -1301,7 +1299,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
/* Read the .gitmodules files from this revision. */ /* Read the .gitmodules files from this revision. */
CanonPath modulesFile(".gitmodules"); CanonPath modulesFile(".gitmodules");
auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, ""); auto accessor = getAccessor(rev, exportIgnore, "");
if (!accessor->pathExists(modulesFile)) if (!accessor->pathExists(modulesFile))
return {}; return {};
@@ -1318,7 +1316,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
std::vector<std::tuple<Submodule, Hash>> result; std::vector<std::tuple<Submodule, Hash>> result;
auto rawAccessor = getRawAccessor(rev, {}); auto rawAccessor = getRawAccessor(rev);
for (auto & submodule : parseSubmodules(pathTemp)) { for (auto & submodule : parseSubmodules(pathTemp)) {
/* Filter out .gitmodules entries that don't exist or are not /* Filter out .gitmodules entries that don't exist or are not
@@ -1334,8 +1332,10 @@ namespace fetchers {
ref<GitRepo> Settings::getTarballCache() const ref<GitRepo> Settings::getTarballCache() const
{ {
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; auto tarballCache(_tarballCache.lock());
return GitRepo::openRepo(repoDir, true, true); if (!*tarballCache)
*tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
return ref<GitRepo>(*tarballCache);
} }
} // namespace fetchers } // namespace fetchers
+20 -54
View File
@@ -433,8 +433,7 @@ struct GitInputScheme : InputScheme
return res; return res;
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Settings & settings, const Input & input, const Path & destDir) const override
const override
{ {
auto repoInfo = getRepoInfo(input); auto repoInfo = getRepoInfo(input);
@@ -778,25 +777,15 @@ struct GitInputScheme : InputScheme
} }
} }
/**
* Decide whether we can do a shallow clone, which is faster. This is possible if the user explicitly specified
* `shallow = true`, or if we already have a `revCount`.
*/
bool canDoShallow(const Input & input) const
{
bool shallow = getShallowAttr(input);
return shallow || input.getRevCount().has_value();
}
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
{ {
assert(!repoInfo.workdirInfo.isDirty); assert(!repoInfo.workdirInfo.isDirty);
auto origRev = input.getRev(); auto origRev = input.getRev();
auto originalRef = input.getRef(); auto originalRef = input.getRef();
bool shallow = canDoShallow(input); bool shallow = getShallowAttr(input);
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
input.attrs.insert_or_assign("ref", ref); input.attrs.insert_or_assign("ref", ref);
@@ -807,27 +796,11 @@ struct GitInputScheme : InputScheme
if (!input.getRev()) if (!input.getRev())
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
} else { } else {
auto rev = input.getRev();
auto repoUrl = std::get<ParsedURL>(repoInfo.location); auto repoUrl = std::get<ParsedURL>(repoInfo.location);
std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow); std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
repoDir = cacheDir; repoDir = cacheDir;
repoInfo.gitDir = "."; repoInfo.gitDir = ".";
/* If shallow = false, but we have a non-shallow repo that already contains the desired rev, then use that
* repo instead. */
std::filesystem::path cacheDirNonShallow = getCachePath(repoUrl.to_string(), false);
if (rev && shallow && pathExists(cacheDirNonShallow)) {
auto nonShallowRepo = GitRepo::openRepo(cacheDirNonShallow, true, true);
if (nonShallowRepo->hasObject(*rev)) {
debug(
"using non-shallow cached repo for '%s' since it contains rev '%s'",
repoUrl.to_string(),
rev->gitRev());
repoDir = cacheDirNonShallow;
goto have_rev;
}
}
std::filesystem::create_directories(cacheDir.parent_path()); std::filesystem::create_directories(cacheDir.parent_path());
PathLocks cacheDirLock({cacheDir.string()}); PathLocks cacheDirLock({cacheDir.string()});
@@ -843,7 +816,7 @@ struct GitInputScheme : InputScheme
/* If a rev was specified, we need to fetch if it's not in the /* If a rev was specified, we need to fetch if it's not in the
repo. */ repo. */
if (rev) { if (auto rev = input.getRev()) {
doFetch = !repo->hasObject(*rev); doFetch = !repo->hasObject(*rev);
} else { } else {
if (getAllRefsAttr(input)) { if (getAllRefsAttr(input)) {
@@ -857,6 +830,7 @@ struct GitInputScheme : InputScheme
} }
if (doFetch) { if (doFetch) {
bool shallow = getShallowAttr(input);
try { try {
auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*" auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*"
: input.getRev() ? input.getRev()->gitRev() : input.getRev() ? input.getRev()->gitRev()
@@ -884,7 +858,7 @@ struct GitInputScheme : InputScheme
warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
} }
if (rev) { if (auto rev = input.getRev()) {
if (!repo->hasObject(*rev)) if (!repo->hasObject(*rev))
throw Error( throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! " "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
@@ -901,30 +875,23 @@ struct GitInputScheme : InputScheme
// the remainder // the remainder
} }
have_rev:
auto repo = GitRepo::openRepo(repoDir); auto repo = GitRepo::openRepo(repoDir);
auto isShallow = repo->isShallow();
if (isShallow && !getShallowAttr(input))
throw Error(
"'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
repoInfo.locationToArg());
// FIXME: check whether rev is an ancestor of ref? // FIXME: check whether rev is an ancestor of ref?
auto rev = *input.getRev(); auto rev = *input.getRev();
/* Skip lastModified computation if it's already supplied by the caller. input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
We don't care if they specify an incorrect value; it doesn't
matter for security, unlike narHash. */
if (!input.attrs.contains("lastModified"))
input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
/* Like lastModified, skip revCount if supplied by the caller. */
if (!shallow && !input.attrs.contains("revCount")) {
auto isShallow = repo->isShallow();
if (isShallow && !shallow)
throw Error(
"'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
repoInfo.locationToArg());
if (!getShallowAttr(input))
input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev)); input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev));
}
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
@@ -932,8 +899,7 @@ struct GitInputScheme : InputScheme
bool exportIgnore = getExportIgnoreAttr(input); bool exportIgnore = getExportIgnoreAttr(input);
bool smudgeLfs = getLfsAttr(input); bool smudgeLfs = getLfsAttr(input);
auto accessor = repo->getAccessor( auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»");
/* If the repo has submodules, fetch them and return a mounted /* If the repo has submodules, fetch them and return a mounted
input accessor consisting of the accessor for the top-level input accessor consisting of the accessor for the top-level
@@ -986,7 +952,7 @@ struct GitInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessorFromWorkdir(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const getAccessorFromWorkdir(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
{ {
auto repoPath = repoInfo.getPath().value(); auto repoPath = repoInfo.getPath().value();
@@ -1000,7 +966,7 @@ struct GitInputScheme : InputScheme
auto exportIgnore = getExportIgnoreAttr(input); auto exportIgnore = getExportIgnoreAttr(input);
ref<SourceAccessor> accessor = ref<SourceAccessor> accessor =
repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath)); repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
/* If the repo has submodules, return a mounted input accessor /* If the repo has submodules, return a mounted input accessor
consisting of the accessor for the top-level repo and the consisting of the accessor for the top-level repo and the
@@ -1070,7 +1036,7 @@ struct GitInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
@@ -1092,7 +1058,7 @@ struct GitInputScheme : InputScheme
return {accessor, std::move(final)}; return {accessor, std::move(final)};
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
auto makeFingerprint = [&](const Hash & rev) { auto makeFingerprint = [&](const Hash & rev) {
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")
+18 -21
View File
@@ -258,7 +258,7 @@ struct GitArchiveInputScheme : InputScheme
std::optional<Hash> treeHash; std::optional<Hash> treeHash;
}; };
virtual RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const = 0; virtual RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0;
@@ -268,7 +268,7 @@ struct GitArchiveInputScheme : InputScheme
time_t lastModified; time_t lastModified;
}; };
std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, Store & store, Input input) const std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, ref<Store> store, Input input) const
{ {
if (!maybeGetStrAttr(input.attrs, "ref")) if (!maybeGetStrAttr(input.attrs, "ref"))
input.attrs.insert_or_assign("ref", "HEAD"); input.attrs.insert_or_assign("ref", "HEAD");
@@ -341,7 +341,7 @@ struct GitArchiveInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{ {
auto [input, tarballInfo] = downloadArchive(settings, store, _input); auto [input, tarballInfo] = downloadArchive(settings, store, _input);
@@ -351,7 +351,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
auto accessor = auto accessor =
settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»"); settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
return {accessor, input}; return {accessor, input};
} }
@@ -370,7 +370,7 @@ struct GitArchiveInputScheme : InputScheme
return Xp::Flakes; return Xp::Flakes;
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto rev = input.getRev()) if (auto rev = input.getRev())
return rev->gitRev(); return rev->gitRev();
@@ -418,7 +418,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
return getStrAttr(input.attrs, "repo"); return getStrAttr(input.attrs, "repo");
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
{ {
auto host = getHost(input); auto host = getHost(input);
auto url = fmt( auto url = fmt(
@@ -432,7 +432,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
auto downloadResult = downloadFile(store, settings, url, "source", headers); auto downloadResult = downloadFile(store, settings, url, "source", headers);
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
return RefInfo{ return RefInfo{
.rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1), .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
@@ -457,13 +457,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Settings & settings, const Input & input, const Path & destDir) const override
const override
{ {
auto host = getHost(input); auto host = getHost(input);
Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(settings, destDir);
} }
}; };
@@ -499,7 +498,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1)); return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1));
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below // See rate limiting note below
@@ -514,7 +513,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
auto downloadResult = downloadFile(store, settings, url, "source", headers); auto downloadResult = downloadFile(store, settings, url, "source", headers);
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)}; return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};
@@ -545,8 +544,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Settings & settings, const Input & input, const Path & destDir) const override
const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere // FIXME: get username somewhere
@@ -554,7 +552,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
settings, settings,
fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(settings, destDir);
} }
}; };
@@ -581,7 +579,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
// Once it is implemented, however, should work as expected. // Once it is implemented, however, should work as expected.
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
{ {
// TODO: In the future, when the sourcehut graphql API is implemented for mercurial // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
// and with anonymous access, this method should use it instead. // and with anonymous access, this method should use it instead.
@@ -597,7 +595,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string refUri; std::string refUri;
if (ref == "HEAD") { if (ref == "HEAD") {
auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers); auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers);
auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
auto remoteLine = git::parseLsRemoteLine(getLine(contents).first); auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
if (!remoteLine) { if (!remoteLine) {
@@ -610,7 +608,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::regex refRegex(refUri); std::regex refRegex(refUri);
auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers); auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers);
auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
std::istringstream is(contents); std::istringstream is(contents);
std::string line; std::string line;
@@ -641,15 +639,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Settings & settings, const Input & input, const Path & destDir) const override
const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
Input::fromURL( Input::fromURL(
settings, settings,
fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(settings, destDir);
} }
}; };
@@ -135,6 +135,8 @@ struct Settings : public Config
private: private:
mutable Sync<std::shared_ptr<Cache>> _cache; mutable Sync<std::shared_ptr<Cache>> _cache;
mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
}; };
} // namespace nix::fetchers } // namespace nix::fetchers
@@ -113,7 +113,7 @@ public:
* Fetch the entire input into the Nix store, returning the * Fetch the entire input into the Nix store, returning the
* location in the Nix store and the locked input. * location in the Nix store and the locked input.
*/ */
std::pair<StorePath, Input> fetchToStore(const Settings & settings, Store & store) const; std::pair<StorePath, Input> fetchToStore(const Settings & settings, ref<Store> store) const;
/** /**
* Check the locking attributes in `result` against * Check the locking attributes in `result` against
@@ -133,17 +133,17 @@ public:
* input without copying it to the store. Also return a possibly * input without copying it to the store. Also return a possibly
* unlocked input. * unlocked input.
*/ */
std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, Store & store) const; std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, ref<Store> store) const;
private: private:
std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, Store & store) const; std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, ref<Store> store) const;
public: public:
Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const; Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const;
void clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const; void clone(const Settings & settings, const Path & destDir) const;
std::optional<std::filesystem::path> getSourcePath() const; std::optional<std::filesystem::path> getSourcePath() const;
@@ -173,7 +173,7 @@ public:
* *
* This is not a stable identifier between Nix versions, but not guaranteed to change either. * This is not a stable identifier between Nix versions, but not guaranteed to change either.
*/ */
std::optional<std::string> getFingerprint(Store & store) const; std::optional<std::string> getFingerprint(ref<Store> store) const;
}; };
/** /**
@@ -229,8 +229,7 @@ struct InputScheme
virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const; virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const;
virtual void virtual void clone(const Settings & settings, const Input & input, const Path & destDir) const;
clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const;
virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const; virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
@@ -241,7 +240,7 @@ struct InputScheme
std::optional<std::string> commitMsg) const; std::optional<std::string> commitMsg) const;
virtual std::pair<ref<SourceAccessor>, Input> virtual std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & input) const = 0; getAccessor(const Settings & settings, ref<Store> store, const Input & input) const = 0;
/** /**
* Is this `InputScheme` part of an experimental feature? * Is this `InputScheme` part of an experimental feature?
@@ -253,7 +252,7 @@ struct InputScheme
return true; return true;
} }
virtual std::optional<std::string> getFingerprint(Store & store, const Input & input) const virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ {
return std::nullopt; return std::nullopt;
} }
@@ -22,12 +22,6 @@ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
virtual Hash flush() = 0; virtual Hash flush() = 0;
}; };
struct GitAccessorOptions
{
bool exportIgnore = false;
bool smudgeLfs = false;
};
struct GitRepo struct GitRepo
{ {
virtual ~GitRepo() {} virtual ~GitRepo() {}
@@ -95,10 +89,10 @@ struct GitRepo
virtual bool hasObject(const Hash & oid) = 0; virtual bool hasObject(const Hash & oid) = 0;
virtual ref<SourceAccessor> virtual ref<SourceAccessor>
getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0; getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) = 0;
virtual ref<SourceAccessor> getAccessor( virtual ref<SourceAccessor>
const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) = 0; getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0; virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
@@ -16,7 +16,7 @@ struct InputCache
}; };
CachedResult CachedResult
getAccessor(const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries); getAccessor(const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
struct CachedInput struct CachedInput
{ {
@@ -13,6 +13,8 @@ namespace nix::fetchers {
struct Registry struct Registry
{ {
const Settings & settings;
enum RegistryType { enum RegistryType {
Flag = 0, Flag = 0,
User = 1, User = 1,
@@ -32,8 +34,9 @@ struct Registry
std::vector<Entry> entries; std::vector<Entry> entries;
Registry(RegistryType type) Registry(const Settings & settings, RegistryType type)
: type{type} : settings{settings}
, type{type}
{ {
} }
@@ -54,9 +57,9 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
Path getUserRegistryPath(); Path getUserRegistryPath();
Registries getRegistries(const Settings & settings, Store & store); Registries getRegistries(const Settings & settings, ref<Store> store);
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs); void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs);
enum class UseRegistries : int { enum class UseRegistries : int {
No, No,
@@ -69,6 +72,6 @@ enum class UseRegistries : int {
* use the registries for which the filter function returns true. * use the registries for which the filter function returns true.
*/ */
std::pair<Input, Attrs> std::pair<Input, Attrs>
lookupInRegistries(const Settings & settings, Store & store, const Input & input, UseRegistries useRegistries); lookupInRegistries(const Settings & settings, ref<Store> store, const Input & input, UseRegistries useRegistries);
} // namespace nix::fetchers } // namespace nix::fetchers
@@ -25,7 +25,7 @@ struct DownloadFileResult
}; };
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
Store & store, ref<Store> store,
const Settings & settings, const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@@ -43,6 +43,6 @@ struct DownloadTarballResult
* Download and import a tarball into the Git cache. The result is the * Download and import a tarball into the Git cache. The result is the
* Git tree hash of the root directory. * Git tree hash of the root directory.
*/ */
ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url); ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url);
} // namespace nix::fetchers } // namespace nix::fetchers
+1 -1
View File
@@ -126,7 +126,7 @@ struct IndirectInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & input) const override
{ {
throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
} }
+1 -1
View File
@@ -6,7 +6,7 @@
namespace nix::fetchers { namespace nix::fetchers {
InputCache::CachedResult InputCache::getAccessor( InputCache::CachedResult InputCache::getAccessor(
const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries) const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
{ {
auto fetched = lookup(originalInput); auto fetched = lookup(originalInput);
Input resolvedInput = originalInput; Input resolvedInput = originalInput;
+10 -10
View File
@@ -179,7 +179,7 @@ struct MercurialInputScheme : InputScheme
return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()}; return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()};
} }
StorePath fetchToStore(const Settings & settings, Store & store, Input & input) const StorePath fetchToStore(const Settings & settings, ref<Store> store, Input & input) const
{ {
auto origRev = input.getRev(); auto origRev = input.getRev();
@@ -230,7 +230,7 @@ struct MercurialInputScheme : InputScheme
return files.count(file); return files.count(file);
}; };
auto storePath = store.addToStore( auto storePath = store->addToStore(
input.getName(), input.getName(),
{getFSSourceAccessor(), CanonPath(actualPath)}, {getFSSourceAccessor(), CanonPath(actualPath)},
ContentAddressMethod::Raw::NixArchive, ContentAddressMethod::Raw::NixArchive,
@@ -251,7 +251,7 @@ struct MercurialInputScheme : InputScheme
"Hash '%s' is not supported by Mercurial. Only sha1 is supported.", "Hash '%s' is not supported by Mercurial. Only sha1 is supported.",
rev.to_string(HashFormat::Base16, true)); rev.to_string(HashFormat::Base16, true));
return Cache::Key{"hgRev", {{"store", store.storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}}; return Cache::Key{"hgRev", {{"store", store->storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
}; };
auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath { auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath {
@@ -271,7 +271,7 @@ struct MercurialInputScheme : InputScheme
/* If we have a rev, check if we have a cached store path. */ /* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), store)) if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
} }
@@ -325,7 +325,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a /* Now that we have the rev, check the cache again for a
cached store path. */ cached store path. */
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), store)) if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
@@ -335,7 +335,7 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir + "/.hg_archival.txt"); deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)}); auto storePath = store->addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
Attrs infoAttrs({ Attrs infoAttrs({
{"revCount", (uint64_t) revCount}, {"revCount", (uint64_t) revCount},
@@ -344,18 +344,18 @@ struct MercurialInputScheme : InputScheme
if (!origRev) if (!origRev)
settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
settings.getCache()->upsert(revInfoKey(rev), store, infoAttrs, storePath); settings.getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath)); return makeResult(infoAttrs, std::move(storePath));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
auto storePath = fetchToStore(settings, store, input); auto storePath = fetchToStore(settings, store, input);
auto accessor = store.requireStoreObjectAccessor(storePath); auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->setPathDisplay("«" + input.to_string() + "»"); accessor->setPathDisplay("«" + input.to_string() + "»");
@@ -367,7 +367,7 @@ struct MercurialInputScheme : InputScheme
return (bool) input.getRev(); return (bool) input.getRev();
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto rev = input.getRev()) if (auto rev = input.getRev())
return rev->gitRev(); return rev->gitRev();
+9 -9
View File
@@ -139,7 +139,7 @@ struct PathInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
auto path = getStrAttr(input.attrs, "path"); auto path = getStrAttr(input.attrs, "path");
@@ -147,31 +147,31 @@ struct PathInputScheme : InputScheme
auto absPath = getAbsPath(input); auto absPath = getAbsPath(input);
// FIXME: check whether access to 'path' is allowed. // FIXME: check whether access to 'path' is allowed.
auto storePath = store.maybeParseStorePath(absPath.string()); auto storePath = store->maybeParseStorePath(absPath.string());
if (storePath) if (storePath)
store.addTempRoot(*storePath); store->addTempRoot(*storePath);
time_t mtime = 0; time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) { if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: try to substitute storePath. // FIXME: try to substitute storePath.
auto src = sinkToSource( auto src = sinkToSource(
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
storePath = store.addToStoreFromDump(*src, "source"); storePath = store->addToStoreFromDump(*src, "source");
} }
auto accessor = store.requireStoreObjectAccessor(*storePath); auto accessor = store->requireStoreObjectAccessor(*storePath);
// To prevent `fetchToStore()` copying the path again to Nix // To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache. // store, pre-create an entry in the fetcher cache.
auto info = store.queryPathInfo(*storePath); auto info = store->queryPathInfo(*storePath);
accessor->fingerprint = accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert( settings.getCache()->upsert(
makeFetchToStoreCacheKey( makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
store, *store,
{}, {},
*storePath); *storePath);
+13 -13
View File
@@ -14,10 +14,10 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
{ {
debug("reading registry '%s'", path); debug("reading registry '%s'", path);
auto registry = std::make_shared<Registry>(type); auto registry = std::make_shared<Registry>(settings, type);
if (!path.pathExists()) if (!path.pathExists())
return std::make_shared<Registry>(type); return std::make_shared<Registry>(settings, type);
try { try {
@@ -125,23 +125,23 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
return customRegistry; return customRegistry;
} }
std::shared_ptr<Registry> getFlagRegistry() std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
{ {
static auto flagRegistry = std::make_shared<Registry>(Registry::Flag); static auto flagRegistry = std::make_shared<Registry>(settings, Registry::Flag);
return flagRegistry; return flagRegistry;
} }
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs) void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs)
{ {
getFlagRegistry()->add(from, to, extraAttrs); getFlagRegistry(settings)->add(from, to, extraAttrs);
} }
static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store) static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
{ {
static auto reg = [&]() { static auto reg = [&]() {
auto path = settings.flakeRegistry.get(); auto path = settings.flakeRegistry.get();
if (path == "") { if (path == "") {
return std::make_shared<Registry>(Registry::Global); // empty registry return std::make_shared<Registry>(settings, Registry::Global); // empty registry
} }
return Registry::read( return Registry::read(
@@ -149,9 +149,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
[&] -> SourcePath { [&] -> SourcePath {
if (!isAbsolute(path)) { if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = dynamic_cast<LocalFSStore *>(&store)) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
return {store.requireStoreObjectAccessor(storePath)}; return {store->requireStoreObjectAccessor(storePath)};
} else { } else {
return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks(); return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
} }
@@ -162,10 +162,10 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
return reg; return reg;
} }
Registries getRegistries(const Settings & settings, Store & store) Registries getRegistries(const Settings & settings, ref<Store> store)
{ {
Registries registries; Registries registries;
registries.push_back(getFlagRegistry()); registries.push_back(getFlagRegistry(settings));
registries.push_back(getUserRegistry(settings)); registries.push_back(getUserRegistry(settings));
registries.push_back(getSystemRegistry(settings)); registries.push_back(getSystemRegistry(settings));
registries.push_back(getGlobalRegistry(settings, store)); registries.push_back(getGlobalRegistry(settings, store));
@@ -173,7 +173,7 @@ Registries getRegistries(const Settings & settings, Store & store)
} }
std::pair<Input, Attrs> std::pair<Input, Attrs>
lookupInRegistries(const Settings & settings, Store & store, const Input & _input, UseRegistries useRegistries) lookupInRegistries(const Settings & settings, ref<Store> store, const Input & _input, UseRegistries useRegistries)
{ {
Attrs extraAttrs; Attrs extraAttrs;
int n = 0; int n = 0;
+12 -12
View File
@@ -13,7 +13,7 @@
namespace nix::fetchers { namespace nix::fetchers {
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
Store & store, ref<Store> store,
const Settings & settings, const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@@ -28,7 +28,7 @@ DownloadFileResult downloadFile(
{"name", name}, {"name", name},
}}}; }}};
auto cached = settings.getCache()->lookupStorePath(key, store); auto cached = settings.getCache()->lookupStorePath(key, *store);
auto useCached = [&]() -> DownloadFileResult { auto useCached = [&]() -> DownloadFileResult {
return { return {
@@ -74,7 +74,7 @@ DownloadFileResult downloadFile(
dumpString(res.data, sink); dumpString(res.data, sink);
auto hash = hashString(HashAlgorithm::SHA256, res.data); auto hash = hashString(HashAlgorithm::SHA256, res.data);
auto info = ValidPathInfo::makeFromCA( auto info = ValidPathInfo::makeFromCA(
store, *store,
name, name,
FixedOutputInfo{ FixedOutputInfo{
.method = FileIngestionMethod::Flat, .method = FileIngestionMethod::Flat,
@@ -84,7 +84,7 @@ DownloadFileResult downloadFile(
hashString(HashAlgorithm::SHA256, sink.s)); hashString(HashAlgorithm::SHA256, sink.s));
info.narSize = sink.s.size(); info.narSize = sink.s.size();
auto source = StringSource{sink.s}; auto source = StringSource{sink.s};
store.addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }
@@ -93,7 +93,7 @@ DownloadFileResult downloadFile(
key.second.insert_or_assign("url", url); key.second.insert_or_assign("url", url);
assert(!res.urls.empty()); assert(!res.urls.empty());
infoAttrs.insert_or_assign("url", *res.urls.rbegin()); infoAttrs.insert_or_assign("url", *res.urls.rbegin());
settings.getCache()->upsert(key, store, infoAttrs, *storePath); settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
} }
return { return {
@@ -136,7 +136,7 @@ static DownloadTarballResult downloadTarball_(
.treeHash = treeHash, .treeHash = treeHash,
.lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
.accessor = settings.getTarballCache()->getAccessor(treeHash, {}, displayPrefix), .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
}; };
}; };
@@ -214,7 +214,7 @@ static DownloadTarballResult downloadTarball_(
return attrsToResult(infoAttrs); return attrsToResult(infoAttrs);
} }
ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url) ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url)
{ {
/* Go through Input::getAccessor() to ensure that the resulting /* Go through Input::getAccessor() to ensure that the resulting
accessor has a fingerprint. */ accessor has a fingerprint. */
@@ -414,7 +414,7 @@ struct FileInputScheme : CurlInputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{ {
auto input(_input); auto input(_input);
@@ -424,10 +424,10 @@ struct FileInputScheme : CurlInputScheme
tarballs. */ tarballs. */
auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName()); auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName());
auto narHash = store.queryPathInfo(file.storePath)->narHash; auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
auto accessor = ref{store.getFSAccessor(file.storePath)}; auto accessor = ref{store->getFSAccessor(file.storePath)};
accessor->setPathDisplay("«" + input.to_string() + "»"); accessor->setPathDisplay("«" + input.to_string() + "»");
@@ -480,7 +480,7 @@ struct TarballInputScheme : CurlInputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
{ {
auto input(_input); auto input(_input);
@@ -505,7 +505,7 @@ struct TarballInputScheme : CurlInputScheme
return {result.accessor, input}; return {result.accessor, input};
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto narHash = input.getNarHash()) if (auto narHash = input.getNarHash())
return narHash->to_string(HashFormat::SRI, true); return narHash->to_string(HashFormat::SRI, true);
+1 -1
View File
@@ -200,7 +200,7 @@ nix_value * nix_locked_flake_get_output_attrs(
nix_clear_err(context); nix_clear_err(context);
try { try {
auto v = nix_alloc_value(context, evalState); auto v = nix_alloc_value(context, evalState);
nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value); nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
return v; return v;
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
+2 -2
View File
@@ -93,7 +93,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
auto & vv = binds.alloc(s); auto & vv = binds.alloc(s);
std::visit( std::visit(
overloaded{ overloaded{
[&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const std::string & value) { vv.mkString(value); },
[&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const uint64_t & value) { vv.mkInt(value); },
[&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }}, [&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }},
value); value);
@@ -156,7 +156,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
} }
} }
auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
v.mkString(flakeRef.to_string(), state.mem); v.mkString(flakeRef.to_string());
} }
nix::PrimOp flakeRefToString({ nix::PrimOp flakeRefToString({
+7 -7
View File
@@ -373,7 +373,7 @@ static Flake getFlake(
{ {
// Fetch a lazy tree first. // Fetch a lazy tree first.
auto cachedInput = auto cachedInput =
state.inputCache->getAccessor(state.fetchSettings, *state.store, originalRef.input, useRegistries); state.inputCache->getAccessor(state.fetchSettings, state.store, originalRef.input, useRegistries);
auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir); auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir); auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
@@ -390,7 +390,7 @@ static Flake getFlake(
// FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
newLockedRef.input.attrs.erase("narHash"); newLockedRef.input.attrs.erase("narHash");
auto cachedInput2 = state.inputCache->getAccessor( auto cachedInput2 = state.inputCache->getAccessor(
state.fetchSettings, *state.store, newLockedRef.input, fetchers::UseRegistries::No); state.fetchSettings, state.store, newLockedRef.input, fetchers::UseRegistries::No);
cachedInput.accessor = cachedInput2.accessor; cachedInput.accessor = cachedInput2.accessor;
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
} }
@@ -756,7 +756,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
return {*resolvedPath, *input.ref}; return {*resolvedPath, *input.ref};
} else { } else {
auto cachedInput = state.inputCache->getAccessor( auto cachedInput = state.inputCache->getAccessor(
state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); state.fetchSettings, state.store, input.ref->input, useRegistriesInputs);
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
@@ -956,7 +956,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
auto key = keyMap.find(node); auto key = keyMap.find(node);
assert(key != keyMap.end()); assert(key != keyMap.end());
override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem); override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel());
overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
} }
@@ -966,7 +966,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix")); Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix"));
auto vLocks = state.allocValue(); auto vLocks = state.allocValue();
vLocks->mkString(lockFileStr, state.mem); vLocks->mkString(lockFileStr);
auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree"); auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
assert(vFetchFinalTree); assert(vFetchFinalTree);
@@ -975,7 +975,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
state.callFunction(*vCallFlake, args, vRes, noPos); state.callFunction(*vCallFlake, args, vRes, noPos);
} }
std::optional<Fingerprint> LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const
{ {
if (lockFile.isUnlocked(fetchSettings)) if (lockFile.isUnlocked(fetchSettings))
return std::nullopt; return std::nullopt;
@@ -1005,7 +1005,7 @@ Flake::~Flake() {}
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake) ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake)
{ {
auto fingerprint = state.settings.useEvalCache && state.settings.pureEval auto fingerprint = state.settings.useEvalCache && state.settings.pureEval
? lockedFlake->getFingerprint(*state.store, state.fetchSettings) ? lockedFlake->getFingerprint(state.store, state.fetchSettings)
: std::nullopt; : std::nullopt;
auto rootLoader = [&state, lockedFlake]() { auto rootLoader = [&state, lockedFlake]() {
/* For testing whether the evaluation cache is /* For testing whether the evaluation cache is
+3 -3
View File
@@ -64,8 +64,8 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef)
return str; return str;
} }
FlakeRef FlakeRef FlakeRef::resolve(
FlakeRef::resolve(const fetchers::Settings & fetchSettings, Store & store, fetchers::UseRegistries useRegistries) const const fetchers::Settings & fetchSettings, ref<Store> store, fetchers::UseRegistries useRegistries) const
{ {
auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries); auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
@@ -289,7 +289,7 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Settings & fetchSettings, const fet
} }
std::pair<ref<SourceAccessor>, FlakeRef> std::pair<ref<SourceAccessor>, FlakeRef>
FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const
{ {
auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store); auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store);
return {accessor, FlakeRef(std::move(lockedInput), subdir)}; return {accessor, FlakeRef(std::move(lockedInput), subdir)};
+1 -1
View File
@@ -135,7 +135,7 @@ struct LockedFlake
*/ */
std::map<ref<Node>, SourcePath> nodePaths; std::map<ref<Node>, SourcePath> nodePaths;
std::optional<Fingerprint> getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; std::optional<Fingerprint> getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const;
}; };
struct LockFlags struct LockFlags
+3 -2
View File
@@ -73,12 +73,13 @@ struct FlakeRef
FlakeRef resolve( FlakeRef resolve(
const fetchers::Settings & fetchSettings, const fetchers::Settings & fetchSettings,
Store & store, ref<Store> store,
fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const; fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs); static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs);
std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const; std::pair<ref<SourceAccessor>, FlakeRef>
lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const;
/** /**
* Canonicalize a flakeref for the purpose of comparing "old" and * Canonicalize a flakeref for the purpose of comparing "old" and
-2
View File
@@ -35,8 +35,6 @@ include_dirs = [ include_directories('.') ]
headers = files( headers = files(
'nix_api_store.h', 'nix_api_store.h',
'nix_api_store/derivation.h',
'nix_api_store/store_path.h',
) )
# TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c`
+36 -2
View File
@@ -12,8 +12,6 @@
*/ */
#include "nix_api_util.h" #include "nix_api_util.h"
#include "nix_api_store/store_path.h"
#include "nix_api_store/derivation.h"
#include <stdbool.h> #include <stdbool.h>
#ifdef __cplusplus #ifdef __cplusplus
@@ -23,6 +21,10 @@ extern "C" {
/** @brief Reference to a Nix store */ /** @brief Reference to a Nix store */
typedef struct Store Store; typedef struct Store Store;
/** @brief Nix store path */
typedef struct StorePath StorePath;
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/** /**
* @brief Initializes the Nix store library * @brief Initializes the Nix store library
@@ -116,6 +118,30 @@ nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_ca
*/ */
StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
/**
* @brief Get the path name (e.g. "name" in /nix/store/...-name)
*
* @param[in] store_path the path to get the name from
* @param[in] callback called with the name
* @param[in] user_data arbitrary data, passed to the callback when it's called.
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
/**
* @brief Copy a StorePath
*
* @param[in] p the path to copy
* @return a new StorePath
*/
StorePath * nix_store_path_clone(const StorePath * p);
/** @brief Deallocate a StorePath
*
* Does not fail.
* @param[in] p the path to free
*/
void nix_store_path_free(StorePath * p);
/** /**
* @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in
* the store) * the store)
@@ -203,6 +229,14 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
*/ */
StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation); StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation);
/**
* @brief Deallocate a `nix_derivation`
*
* Does not fail.
* @param[in] drv the derivation to free
*/
void nix_derivation_free(nix_derivation * drv);
/** /**
* @brief Copy the closure of `path` from `srcStore` to `dstStore`. * @brief Copy the closure of `path` from `srcStore` to `dstStore`.
* *
-38
View File
@@ -1,38 +0,0 @@
#ifndef NIX_API_STORE_DERIVATION_H
#define NIX_API_STORE_DERIVATION_H
/**
* @defgroup libstore_derivation Derivation
* @ingroup libstore
* @brief Derivation operations that don't require a Store
* @{
*/
/** @file
* @brief Derivation operations
*/
#include "nix_api_util.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/**
* @brief Deallocate a `nix_derivation`
*
* Does not fail.
* @param[in] drv the derivation to free
*/
void nix_derivation_free(nix_derivation * drv);
// cffi end
#ifdef __cplusplus
}
#endif
/**
* @}
*/
#endif // NIX_API_STORE_DERIVATION_H
-54
View File
@@ -1,54 +0,0 @@
#ifndef NIX_API_STORE_STORE_PATH_H
#define NIX_API_STORE_STORE_PATH_H
/**
* @defgroup libstore_storepath StorePath
* @ingroup libstore
* @brief Store path operations that don't require a Store
* @{
*/
/** @file
* @brief Store path operations
*/
#include "nix_api_util.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
/** @brief Nix store path */
typedef struct StorePath StorePath;
/**
* @brief Copy a StorePath
*
* @param[in] p the path to copy
* @return a new StorePath
*/
StorePath * nix_store_path_clone(const StorePath * p);
/** @brief Deallocate a StorePath
*
* Does not fail.
* @param[in] p the path to free
*/
void nix_store_path_free(StorePath * p);
/**
* @brief Get the path name (e.g. "<name>" in /nix/store/<hash>-<name>)
*
* @param[in] store_path the path to get the name from
* @param[in] callback called with the name
* @param[in] user_data arbitrary data, passed to the callback when it's called.
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
// cffi end
#ifdef __cplusplus
}
#endif
/**
* @}
*/
#endif // NIX_API_STORE_STORE_PATH_H
@@ -4,13 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"refs2": [ "refs2": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -23,36 +20,18 @@
"outputChecks": { "outputChecks": {
"forAllOutputs": { "forAllOutputs": {
"allowedReferences": [ "allowedReferences": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
"drvPath": "self", "bin"
"output": "bin"
},
{
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "dev"
}
], ],
"disallowedReferences": [ "disallowedReferences": [
{ "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
"drvPath": "self", "dev"
"output": "dev"
},
{
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "out"
}
], ],
"disallowedRequisites": [ "disallowedRequisites": [
{ "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "dev"
}
], ],
"ignoreSelfRefs": true, "ignoreSelfRefs": true,
"maxClosureSize": null, "maxClosureSize": null,
@@ -4,13 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"refs2": [ "refs2": [
"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -26,20 +23,11 @@
"allowedReferences": null, "allowedReferences": null,
"allowedRequisites": null, "allowedRequisites": null,
"disallowedReferences": [ "disallowedReferences": [
{ "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
"drvPath": "self", "dev"
"output": "dev"
},
{
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "out"
}
], ],
"disallowedRequisites": [ "disallowedRequisites": [
{ "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
"drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
"output": "dev"
}
], ],
"ignoreSelfRefs": false, "ignoreSelfRefs": false,
"maxClosureSize": null, "maxClosureSize": null,
@@ -56,20 +44,11 @@
}, },
"out": { "out": {
"allowedReferences": [ "allowedReferences": [
{ "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "out"
}
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
"drvPath": "self", "bin"
"output": "bin"
},
{
"drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
"output": "dev"
}
], ],
"disallowedReferences": [], "disallowedReferences": [],
"disallowedRequisites": [], "disallowedRequisites": [],
@@ -4,10 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"refs2": [ "refs2": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -20,24 +20,18 @@
"outputChecks": { "outputChecks": {
"forAllOutputs": { "forAllOutputs": {
"allowedReferences": [ "allowedReferences": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
"drvPath": "self", "bin"
"output": "bin"
},
"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
], ],
"disallowedReferences": [ "disallowedReferences": [
{ "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
"drvPath": "self", "dev"
"output": "dev"
},
"r5cff30838majxk5mp3ip2diffi8vpaj-bar"
], ],
"disallowedRequisites": [ "disallowedRequisites": [
"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
], ],
"ignoreSelfRefs": true, "ignoreSelfRefs": true,
"maxClosureSize": null, "maxClosureSize": null,
@@ -4,10 +4,10 @@
"allowSubstitutes": false, "allowSubstitutes": false,
"exportReferencesGraph": { "exportReferencesGraph": {
"refs1": [ "refs1": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"refs2": [ "refs2": [
"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
] ]
}, },
"impureEnvVars": [ "impureEnvVars": [
@@ -23,14 +23,11 @@
"allowedReferences": null, "allowedReferences": null,
"allowedRequisites": null, "allowedRequisites": null,
"disallowedReferences": [ "disallowedReferences": [
{ "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
"drvPath": "self", "dev"
"output": "dev"
},
"r5cff30838majxk5mp3ip2diffi8vpaj-bar"
], ],
"disallowedRequisites": [ "disallowedRequisites": [
"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
], ],
"ignoreSelfRefs": false, "ignoreSelfRefs": false,
"maxClosureSize": null, "maxClosureSize": null,
@@ -47,14 +44,11 @@
}, },
"out": { "out": {
"allowedReferences": [ "allowedReferences": [
"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
], ],
"allowedRequisites": [ "allowedRequisites": [
{ "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
"drvPath": "self", "bin"
"output": "bin"
},
"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
], ],
"disallowedReferences": [], "disallowedReferences": [],
"disallowedRequisites": [], "disallowedRequisites": [],
+69 -96
View File
@@ -3,7 +3,7 @@
#include "nix/util/experimental-features.hh" #include "nix/util/experimental-features.hh"
#include "nix/store/derivations.hh" #include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh" #include "nix/store/derivations.hh"
#include "nix/store/derivation-options.hh" #include "nix/store/derivation-options.hh"
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/util/types.hh" #include "nix/util/types.hh"
@@ -17,7 +17,7 @@ namespace nix {
using namespace nlohmann; using namespace nlohmann;
class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>, class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
public JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>, public JsonCharacterizationTest<DerivationOptions>,
public LibStoreTest public LibStoreTest
{ {
protected: protected:
@@ -42,8 +42,7 @@ public:
{ {
this->readTest(fileName, [&](auto encoded) { this->readTest(fileName, [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures);
}); });
} }
@@ -52,14 +51,11 @@ public:
* Helper function to test DerivationOptions parsing and comparison * Helper function to test DerivationOptions parsing and comparison
*/ */
void testDerivationOptions( void testDerivationOptions(
const std::string & fileName, const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures)
const DerivationOptions<SingleDerivedPath> & expected,
const StringSet & expectedSystemFeatures)
{ {
this->readTest(fileName, [&](auto encoded) { this->readTest(fileName, [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_EQ(options, expected); EXPECT_EQ(options, expected);
EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures); EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures);
@@ -135,38 +131,22 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute
* Since these are both repeated and sensative opaque values, it makes * Since these are both repeated and sensative opaque values, it makes
* sense to give them names in this file. * sense to give them names in this file.
*/ */
static SingleDerivedPath static std::string pathFoo = "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
pathFoo = SingleDerivedPath::Opaque{StorePath{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}}, pathFooDev = "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
pathFooDev = SingleDerivedPath::Opaque{StorePath{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}}, pathBar = "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
pathBar = SingleDerivedPath::Opaque{StorePath{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}}, pathBarDev = "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
pathBarDev = SingleDerivedPath::Opaque{StorePath{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}}, pathBarDrvIA = "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
pathBarDrvIA = SingleDerivedPath::Opaque{StorePath{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}}, pathBarDrvCA = "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
pathBarDrvCA = SingleDerivedPath::Opaque{StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}}, placeholderFoo = "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
placeholderFoo = placeholderFooDev = "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
SingleDerivedPath::Built{ placeholderBar = "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
.drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}), placeholderBarDev = "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8";
.output = "out",
},
placeholderFooDev =
SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
.output = "dev",
},
placeholderBar =
SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
.output = "out",
},
placeholderBarDev = SingleDerivedPath::Built{
.drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
.output = "dev",
};
using ExportReferencesMap = decltype(DerivationOptions<SingleDerivedPath>::exportReferencesGraph); using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph);
static const DerivationOptions<SingleDerivedPath> advancedAttributes_defaults = { static const DerivationOptions advancedAttributes_defaults = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
@@ -187,8 +167,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(!got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs);
@@ -213,9 +192,9 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
{ {
DerivationOptions<SingleDerivedPath> expected = { DerivationOptions expected = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
@@ -233,13 +212,12 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
this->readTest("advanced-attributes.drv", [&](auto encoded) { this->readTest("advanced-attributes.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(!got.structuredAttrs); EXPECT_TRUE(!got.structuredAttrs);
// Reset fields that vary between test cases to enable whole-object comparison // Reset fields that vary between test cases to enable whole-object comparison
options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true}; options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true};
options.exportReferencesGraph = {}; options.exportReferencesGraph = {};
EXPECT_EQ(options, expected); EXPECT_EQ(options, expected);
@@ -249,14 +227,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
}); });
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_ia = { DerivationOptions advancedAttributes_ia = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo}, .allowedReferences = StringSet{pathFoo},
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}}, .disallowedReferences = StringSet{pathBar, "dev"},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{pathFooDev, "bin"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev}, .disallowedRequisites = StringSet{pathBarDev},
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
.passAsFile = {}, .passAsFile = {},
@@ -279,14 +257,14 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia)
testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"}); testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"});
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_ca = { DerivationOptions advancedAttributes_ca = {
.outputChecks = .outputChecks =
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo}, .allowedReferences = StringSet{placeholderFoo},
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}}, .disallowedReferences = StringSet{placeholderBar, "dev"},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{placeholderFooDev, "bin"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev}, .disallowedRequisites = StringSet{placeholderBarDev},
}, },
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
.passAsFile = {}, .passAsFile = {},
@@ -309,8 +287,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"}); testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"});
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_defaults = { DerivationOptions advancedAttributes_structuredAttrs_defaults = {
.outputChecks = std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{}, .outputChecks = std::map<std::string, DerivationOptions::OutputChecks>{},
.unsafeDiscardReferences = {}, .unsafeDiscardReferences = {},
.passAsFile = {}, .passAsFile = {},
.exportReferencesGraph = {}, .exportReferencesGraph = {},
@@ -329,8 +307,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs);
@@ -355,11 +332,11 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
{ {
DerivationOptions<SingleDerivedPath> expected = { DerivationOptions expected = {
.outputChecks = .outputChecks =
std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{ std::map<std::string, DerivationOptions::OutputChecks>{
{"dev", {"dev",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.maxSize = 789, .maxSize = 789,
.maxClosureSize = 5909, .maxClosureSize = 5909,
}}, }},
@@ -380,8 +357,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
auto options = derivationOptionsFromStructuredAttrs( DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
*this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
EXPECT_TRUE(got.structuredAttrs); EXPECT_TRUE(got.structuredAttrs);
@@ -389,8 +365,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
{ {
// Delete all keys but "dev" in options.outputChecks // Delete all keys but "dev" in options.outputChecks
auto * outputChecksMapP = auto * outputChecksMapP =
std::get_if<std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>>( std::get_if<std::map<std::string, DerivationOptions::OutputChecks>>(&options.outputChecks);
&options.outputChecks);
ASSERT_TRUE(outputChecksMapP); ASSERT_TRUE(outputChecksMapP);
auto & outputChecksMap = *outputChecksMapP; auto & outputChecksMap = *outputChecksMapP;
auto devEntry = outputChecksMap.find("dev"); auto devEntry = outputChecksMap.find("dev");
@@ -410,21 +385,21 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
}); });
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ia = { DerivationOptions advancedAttributes_structuredAttrs_ia = {
.outputChecks = .outputChecks =
std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{ std::map<std::string, DerivationOptions::OutputChecks>{
{"out", {"out",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo}, .allowedReferences = StringSet{pathFoo},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{pathFooDev, "bin"},
}}, }},
{"bin", {"bin",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}}, .disallowedReferences = StringSet{pathBar, "dev"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev}, .disallowedRequisites = StringSet{pathBarDev},
}}, }},
{"dev", {"dev",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.maxSize = 789, .maxSize = 789,
.maxClosureSize = 5909, .maxClosureSize = 5909,
}}, }},
@@ -452,21 +427,21 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
"advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"}); "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"});
}; };
DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ca = { DerivationOptions advancedAttributes_structuredAttrs_ca = {
.outputChecks = .outputChecks =
std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{ std::map<std::string, DerivationOptions::OutputChecks>{
{"out", {"out",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo}, .allowedReferences = StringSet{placeholderFoo},
.allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}}, .allowedRequisites = StringSet{placeholderFooDev, "bin"},
}}, }},
{"bin", {"bin",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}}, .disallowedReferences = StringSet{placeholderBar, "dev"},
.disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev}, .disallowedRequisites = StringSet{placeholderBarDev},
}}, }},
{"dev", {"dev",
DerivationOptions<SingleDerivedPath>::OutputChecks{ DerivationOptions::OutputChecks{
.maxSize = 789, .maxSize = 789,
.maxClosureSize = 5909, .maxClosureSize = 5909,
}}, }},
@@ -496,16 +471,14 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
{"rainbow", "uid-range", "ca-derivations"}); {"rainbow", "uid-range", "ca-derivations"});
}; };
#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \ #define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2) \
TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \ TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json) \
{ \ { \
this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::readJsonTest( \ this->JsonCharacterizationTest<DerivationOptions>::readJsonTest(#VAR, advancedAttributes_##VAR2); \
#VAR, advancedAttributes_##VAR2); \ } \
} \ TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \
TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json) \ { \
{ \ this->JsonCharacterizationTest<DerivationOptions>::writeJsonTest(#VAR, advancedAttributes_##VAR2); \
this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::writeJsonTest( \
#VAR, advancedAttributes_##VAR2); \
} }
TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults) TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults)
+2 -2
View File
@@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep)
// Create an in-memory file system with various reference patterns // Create an in-memory file system with various reference patterns
auto accessor = make_ref<MemorySourceAccessor>(); auto accessor = make_ref<MemorySourceAccessor>();
accessor->root = File::Directory{ accessor->root = File::Directory{
.entries{ .contents{
{ {
// file1.txt: contains hash1 // file1.txt: contains hash1
"file1.txt", "file1.txt",
@@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep)
// subdir: a subdirectory // subdir: a subdirectory
"subdir", "subdir",
File::Directory{ File::Directory{
.entries{ .contents{
{ {
// subdir/file4.txt: contains hash1 again // subdir/file4.txt: contains hash1 again
"file4.txt", "file4.txt",
+2 -2
View File
@@ -8,7 +8,7 @@
#include "nix/util/sync.hh" #include "nix/util/sync.hh"
#include "nix/store/remote-fs-accessor.hh" #include "nix/store/remote-fs-accessor.hh"
#include "nix/store/nar-info-disk-cache.hh" #include "nix/store/nar-info-disk-cache.hh"
#include "nix/util/nar-accessor.hh" #include "nix/store/nar-accessor.hh"
#include "nix/util/thread-pool.hh" #include "nix/util/thread-pool.hh"
#include "nix/util/callback.hh" #include "nix/util/callback.hh"
#include "nix/util/signals.hh" #include "nix/util/signals.hh"
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
if (config.writeNARListing) { if (config.writeNARListing) {
nlohmann::json j = { nlohmann::json j = {
{"version", 1}, {"version", 1},
{"root", listNarDeep(*narAccessor, CanonPath::root)}, {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
}; };
upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
+16 -41
View File
@@ -32,6 +32,14 @@ DerivationBuildingGoal::DerivationBuildingGoal(
, drv{std::make_unique<Derivation>(drv)} , drv{std::make_unique<Derivation>(drv)}
, buildMode(buildMode) , buildMode(buildMode)
{ {
try {
drvOptions =
std::make_unique<DerivationOptions>(DerivationOptions::fromStructuredAttrs(drv.env, drv.structuredAttrs));
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath)); name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath));
trace("created"); trace("created");
@@ -198,38 +206,6 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
Goal::Co DerivationBuildingGoal::tryToBuild() Goal::Co DerivationBuildingGoal::tryToBuild()
{ {
auto drvOptions = [&] {
DerivationOptions<SingleDerivedPath> temp;
try {
temp =
derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
} catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw;
}
auto res = tryResolve(
temp,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
try {
return resolveDerivedPath(
worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore);
} catch (Error &) {
return std::nullopt;
}
});
/* The derivation must have all of its inputs gotten this point,
so the resolution will surely succeed.
(Actually, we shouldn't even enter this goal until we have a
resolved derivation, or derivation with only input addressed
transitive inputs, so this should be a no-opt anyways.)
*/
assert(res);
return *res;
}();
std::map<std::string, InitialOutput> initialOutputs; std::map<std::string, InitialOutput> initialOutputs;
/* Recheck at this point. In particular, whereas before we were /* Recheck at this point. In particular, whereas before we were
@@ -368,13 +344,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
/* Don't do a remote build if the derivation has the attribute /* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only `preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */ supported for local builds. */
bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv)) bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
&& settings.maxBuildJobs.get() != 0; && settings.maxBuildJobs.get() != 0;
if (buildLocally) { if (buildLocally) {
useHook = false; useHook = false;
} else { } else {
switch (tryBuildHook(initialOutputs, drvOptions)) { switch (tryBuildHook(initialOutputs)) {
case rpAccept: case rpAccept:
/* Yes, it has started doing so. Wait until we get /* Yes, it has started doing so. Wait until we get
EOF from the hook. */ EOF from the hook. */
@@ -403,7 +379,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv); externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) { if (!externalBuilder && !drvOptions->canBuildLocally(worker.store, *drv)) {
auto msg = auto msg =
fmt("Cannot build '%s'.\n" fmt("Cannot build '%s'.\n"
"Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL "Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
@@ -412,7 +388,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
"Current system: '%s' with features {%s}", "Current system: '%s' with features {%s}",
Magenta(worker.store.printStorePath(drvPath)), Magenta(worker.store.printStorePath(drvPath)),
Magenta(drv->platform), Magenta(drv->platform),
concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(*drv)), concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
Magenta(settings.thisSystem), Magenta(settings.thisSystem),
concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures)); concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures));
@@ -610,7 +586,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
} }
try { try {
desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths); desugaredEnv = DesugaredEnv::create(worker.store, *drv, *drvOptions, inputPaths);
} catch (BuildError & e) { } catch (BuildError & e) {
outputLocks.unlock(); outputLocks.unlock();
worker.permanentFailure = true; worker.permanentFailure = true;
@@ -621,7 +597,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
.drvPath = drvPath, .drvPath = drvPath,
.buildResult = buildResult, .buildResult = buildResult,
.drv = *drv, .drv = *drv,
.drvOptions = drvOptions, .drvOptions = *drvOptions,
.inputPaths = inputPaths, .inputPaths = inputPaths,
.initialOutputs = initialOutputs, .initialOutputs = initialOutputs,
.buildMode = buildMode, .buildMode = buildMode,
@@ -827,8 +803,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
return BuildError{e.status, msg}; return BuildError{e.status, msg};
} }
HookReply DerivationBuildingGoal::tryBuildHook( HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs)
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions)
{ {
#ifdef _WIN32 // TODO enable build hook on Windows #ifdef _WIN32 // TODO enable build hook on Windows
return rpDecline; return rpDecline;
@@ -845,7 +820,7 @@ HookReply DerivationBuildingGoal::tryBuildHook(
/* Send the request to the hook. */ /* Send the request to the hook. */
worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
<< worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv); << worker.store.printStorePath(drvPath) << drvOptions->getRequiredSystemFeatures(*drv);
worker.hook->sink.flush(); worker.hook->sink.flush();
/* Read the first line of input, which should be a word indicating /* Read the first line of input, which should be a word indicating
+21 -26
View File
@@ -11,7 +11,7 @@ void checkOutputs(
Store & store, Store & store,
const StorePath & drvPath, const StorePath & drvPath,
const decltype(Derivation::outputs) & drvOutputs, const decltype(Derivation::outputs) & drvOutputs,
const decltype(DerivationOptions<StorePath>::outputChecks) & outputChecks, const decltype(DerivationOptions::outputChecks) & outputChecks,
const std::map<std::string, ValidPathInfo> & outputs) const std::map<std::string, ValidPathInfo> & outputs)
{ {
std::map<Path, const ValidPathInfo &> outputsByPath; std::map<Path, const ValidPathInfo &> outputsByPath;
@@ -85,7 +85,7 @@ void checkOutputs(
return std::make_pair(std::move(pathsDone), closureSize); return std::make_pair(std::move(pathsDone), closureSize);
}; };
auto applyChecks = [&](const DerivationOptions<StorePath>::OutputChecks & checks) { auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) {
if (checks.maxSize && info.narSize > *checks.maxSize) if (checks.maxSize && info.narSize > *checks.maxSize)
throw BuildError( throw BuildError(
BuildResult::Failure::OutputRejected, BuildResult::Failure::OutputRejected,
@@ -105,33 +105,28 @@ void checkOutputs(
*checks.maxClosureSize); *checks.maxClosureSize);
} }
auto checkRefs = [&](const std::set<DrvRef<StorePath>> & value, bool allowed, bool recursive) { auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) {
/* Parse a list of reference specifiers. Each element must /* Parse a list of reference specifiers. Each element must
either be a store path, or the symbolic name of the output either be a store path, or the symbolic name of the output
of the derivation (such as `out'). */ of the derivation (such as `out'). */
StorePathSet spec; StorePathSet spec;
for (auto & i : value) { for (auto & i : value) {
std::visit( if (store.isStorePath(i))
overloaded{ spec.insert(store.parseStorePath(i));
[&](const StorePath & path) { spec.insert(path); }, else if (auto output = get(outputs, i))
[&](const OutputName & refOutputName) { spec.insert(output->path);
if (auto output = get(outputs, refOutputName)) else {
spec.insert(output->path); std::string outputsListing =
else { concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
std::string outputsListing = throw BuildError(
concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); BuildResult::Failure::OutputRejected,
throw BuildError( "derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
BuildResult::Failure::OutputRejected, " expected store path or output name (one of [%s])",
"derivation '%s' output check for '%s' contains output name '%s'," store.printStorePath(drvPath),
" but this is not a valid output of this derivation." outputName,
" (Valid outputs are [%s].)", i,
store.printStorePath(drvPath), outputsListing);
outputName, }
refOutputName,
outputsListing);
}
}},
i);
} }
auto used = recursive ? getClosure(info.path).first : info.references; auto used = recursive ? getClosure(info.path).first : info.references;
@@ -185,8 +180,8 @@ void checkOutputs(
std::visit( std::visit(
overloaded{ overloaded{
[&](const DerivationOptions<StorePath>::OutputChecks & checks) { applyChecks(checks); }, [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); },
[&](const std::map<std::string, DerivationOptions<StorePath>::OutputChecks> & checksPerOutput) { [&](const std::map<std::string, DerivationOptions::OutputChecks> & checksPerOutput) {
if (auto outputChecks = get(checksPerOutput, outputName)) if (auto outputChecks = get(checksPerOutput, outputName))
applyChecks(*outputChecks); applyChecks(*outputChecks);
+1 -1
View File
@@ -21,7 +21,7 @@ void checkOutputs(
Store & store, Store & store,
const StorePath & drvPath, const StorePath & drvPath,
const decltype(Derivation::outputs) & drvOutputs, const decltype(Derivation::outputs) & drvOutputs,
const decltype(DerivationOptions<StorePath>::outputChecks) & drvOptions, const decltype(DerivationOptions::outputChecks) & drvOptions,
const std::map<std::string, ValidPathInfo> & outputs); const std::map<std::string, ValidPathInfo> & outputs);
} // namespace nix } // namespace nix
+2 -5
View File
@@ -18,10 +18,7 @@ std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fil
} }
DesugaredEnv DesugaredEnv::create( DesugaredEnv DesugaredEnv::create(
Store & store, Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths)
const Derivation & drv,
const DerivationOptions<StorePath> & drvOptions,
const StorePathSet & inputPaths)
{ {
DesugaredEnv res; DesugaredEnv res;
@@ -49,7 +46,7 @@ DesugaredEnv DesugaredEnv::create(
} }
/* Handle exportReferencesGraph(), if set. */ /* Handle exportReferencesGraph(), if set. */
for (auto & [fileName, storePaths] : drvOptions.exportReferencesGraph) { for (auto & [fileName, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
/* Write closure info to <fileName>. */ /* Write closure info to <fileName>. */
res.extraFiles.insert_or_assign( res.extraFiles.insert_or_assign(
fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false)); fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false));
+2 -3
View File
@@ -64,10 +64,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
{ {
trace("have derivation"); trace("have derivation");
auto drvOptions = [&]() -> DerivationOptions<SingleDerivedPath> { auto drvOptions = [&]() -> DerivationOptions {
try { try {
return derivationOptionsFromStructuredAttrs( return DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw; throw;
+67 -298
View File
@@ -2,18 +2,15 @@
#include "nix/util/json-utils.hh" #include "nix/util/json-utils.hh"
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/store/derivations.hh" #include "nix/store/derivations.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/util.hh" #include "nix/util/util.hh"
#include "nix/store/globals.hh" #include "nix/store/globals.hh"
#include "nix/util/variant-wrapper.hh"
#include <optional> #include <optional>
#include <string> #include <string>
#include <variant> #include <variant>
#include <regex> #include <regex>
#include <ranges>
namespace nix { namespace nix {
@@ -93,38 +90,14 @@ getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const st
return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{}); return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
} }
template<typename Inputs> using OutputChecks = DerivationOptions::OutputChecks;
using OutputChecks = DerivationOptions<Inputs>::OutputChecks;
template<typename Inputs> using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
using OutputChecksVariant = std::variant<OutputChecks<Inputs>, std::map<std::string, OutputChecks<Inputs>>>;
DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs( DerivationOptions DerivationOptions::fromStructuredAttrs(
const StoreDirConfig & store, const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn)
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn,
const ExperimentalFeatureSettings & mockXpSettings)
{ {
/* Use the SingleDerivedPath version with empty inputDrvs, then return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
resolve. */
DerivedPathMap<StringSet> emptyInputDrvs{};
auto singleDerivedPathOptions =
derivationOptionsFromStructuredAttrs(store, emptyInputDrvs, env, parsed, shouldWarn, mockXpSettings);
/* "Resolve" all SingleDerivedPath inputs to StorePath. */
auto resolved = tryResolve(
singleDerivedPathOptions,
[&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
// there should be nothing to resolve
assert(false);
});
/* Since we should never need to call the call back, there should be
no way it fails. */
assert(resolved);
return *resolved;
} }
static void flatten(const nlohmann::json & value, StringSet & res) static void flatten(const nlohmann::json & value, StringSet & res)
@@ -138,63 +111,10 @@ static void flatten(const nlohmann::json & value, StringSet & res)
throw Error("'exportReferencesGraph' value is not an array or a string"); throw Error("'exportReferencesGraph' value is not an array or a string");
} }
DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs( DerivationOptions
const StoreDirConfig & store, DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
const DerivedPathMap<StringSet> & inputDrvs,
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn,
const ExperimentalFeatureSettings & mockXpSettings)
{ {
DerivationOptions<SingleDerivedPath> defaults = {}; DerivationOptions defaults = {};
std::map<std::string, SingleDerivedPath::Built> placeholders;
if (mockXpSettings.isEnabled(Xp::CaDerivations)) {
/* Initialize placeholder map from inputDrvs */
auto initPlaceholders = [&](this const auto & initPlaceholders,
ref<const SingleDerivedPath> basePath,
const DerivedPathMap<StringSet>::ChildNode & node) -> void {
for (const auto & outputName : node.value) {
auto built = SingleDerivedPath::Built{
.drvPath = basePath,
.output = outputName,
};
placeholders.insert_or_assign(
DownstreamPlaceholder::fromSingleDerivedPathBuilt(built, mockXpSettings).render(),
std::move(built));
}
for (const auto & [outputName, childNode] : node.childMap) {
initPlaceholders(
make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
.drvPath = basePath,
.output = outputName,
}),
childNode);
}
};
for (const auto & [drvPath, outputs] : inputDrvs.map) {
auto basePath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{drvPath});
initPlaceholders(basePath, outputs);
}
}
auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath {
if (auto it = placeholders.find(pathS); it != placeholders.end())
return it->second;
else
return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
};
auto parseRef = [&](const std::string & pathS) -> DrvRef<SingleDerivedPath> {
if (auto it = placeholders.find(pathS); it != placeholders.end())
return it->second;
if (store.isStorePath(pathS))
return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
else
return pathS;
};
if (shouldWarn && parsed) { if (shouldWarn && parsed) {
auto & structuredAttrs = parsed->structuredAttrs; auto & structuredAttrs = parsed->structuredAttrs;
@@ -226,14 +146,14 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
} }
return { return {
.outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> { .outputChecks = [&]() -> OutputChecksVariant {
if (parsed) { if (parsed) {
auto & structuredAttrs = parsed->structuredAttrs; auto & structuredAttrs = parsed->structuredAttrs;
std::map<std::string, OutputChecks<SingleDerivedPath>> res; std::map<std::string, OutputChecks> res;
if (auto * outputChecks = get(structuredAttrs, "outputChecks")) { if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
for (auto & [outputName, output_] : getObject(*outputChecks)) { for (auto & [outputName, output_] : getObject(*outputChecks)) {
OutputChecks<SingleDerivedPath> checks; OutputChecks checks;
auto & output = getObject(output_); auto & output = getObject(output_);
@@ -243,14 +163,13 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
if (auto maxClosureSize = get(output, "maxClosureSize")) if (auto maxClosureSize = get(output, "maxClosureSize"))
checks.maxClosureSize = maxClosureSize->get<uint64_t>(); checks.maxClosureSize = maxClosureSize->get<uint64_t>();
auto get_ = auto get_ = [&output = output](const std::string & name) -> std::optional<StringSet> {
[&](const std::string & name) -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
if (auto i = get(output, name)) { if (auto i = get(output, name)) {
std::set<DrvRef<SingleDerivedPath>> res; StringSet res;
for (auto j = i->begin(); j != i->end(); ++j) { for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string()) if (!j->is_string())
throw Error("attribute '%s' must be a list of strings", name); throw Error("attribute '%s' must be a list of strings", name);
res.insert(parseRef(j->get<std::string>())); res.insert(j->get<std::string>());
} }
return res; return res;
} }
@@ -259,7 +178,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
res.insert_or_assign( res.insert_or_assign(
outputName, outputName,
OutputChecks<SingleDerivedPath>{ OutputChecks{
.maxSize = [&]() -> std::optional<uint64_t> { .maxSize = [&]() -> std::optional<uint64_t> {
if (auto maxSize = get(output, "maxSize")) if (auto maxSize = get(output, "maxSize"))
return maxSize->get<uint64_t>(); return maxSize->get<uint64_t>();
@@ -273,32 +192,21 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
return std::nullopt; return std::nullopt;
}(), }(),
.allowedReferences = get_("allowedReferences"), .allowedReferences = get_("allowedReferences"),
.disallowedReferences = .disallowedReferences = get_("disallowedReferences").value_or(StringSet{}),
get_("disallowedReferences").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
.allowedRequisites = get_("allowedRequisites"), .allowedRequisites = get_("allowedRequisites"),
.disallowedRequisites = .disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}),
get_("disallowedRequisites").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
}); });
} }
} }
return res; return res;
} else { } else {
auto parseRefSet = [&](const std::optional<StringSet> optionalStringSet) return OutputChecks{
-> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
if (!optionalStringSet)
return std::nullopt;
auto range = *optionalStringSet | std::views::transform(parseRef);
return std::set<DrvRef<SingleDerivedPath>>(range.begin(), range.end());
};
return OutputChecks<SingleDerivedPath>{
// legacy non-structured-attributes case // legacy non-structured-attributes case
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = parseRefSet(getStringSetAttr(env, parsed, "allowedReferences")), .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"),
.disallowedReferences = parseRefSet(getStringSetAttr(env, parsed, "disallowedReferences")) .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}),
.value_or(std::set<DrvRef<SingleDerivedPath>>{}), .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"),
.allowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "allowedRequisites")), .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}),
.disallowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "disallowedRequisites"))
.value_or(std::set<DrvRef<SingleDerivedPath>>{}),
}; };
} }
}(), }(),
@@ -337,19 +245,16 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
}(), }(),
.exportReferencesGraph = .exportReferencesGraph =
[&] { [&] {
std::map<std::string, std::set<SingleDerivedPath>> ret; std::map<std::string, StringSet> ret;
if (parsed) { if (parsed) {
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph"); auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
if (!e || !e->is_object()) if (!e || !e->is_object())
return ret; return ret;
for (auto & [key, storePathsJson] : getObject(*e)) { for (auto & [key, value] : getObject(*e)) {
StringSet ss; StringSet ss;
flatten(storePathsJson, ss); flatten(value, ss);
std::set<SingleDerivedPath> storePaths; ret.insert_or_assign(key, std::move(ss));
for (auto & s : ss)
storePaths.insert(parseSingleDerivedPath(s));
ret.insert_or_assign(key, std::move(storePaths));
} }
} else { } else {
auto s = getOr(env, "exportReferencesGraph", ""); auto s = getOr(env, "exportReferencesGraph", "");
@@ -363,7 +268,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
auto & storePathS = *i++; auto & storePathS = *i++;
ret.insert_or_assign(std::move(fileName), std::set{parseSingleDerivedPath(storePathS)}); ret.insert_or_assign(std::move(fileName), StringSet{storePathS});
} }
} }
return ret; return ret;
@@ -381,8 +286,28 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
}; };
} }
template<typename Input> std::map<std::string, StorePathSet>
StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivation & drv) const DerivationOptions::getParsedExportReferencesGraph(const StoreDirConfig & store) const
{
std::map<std::string, StorePathSet> res;
for (auto & [fileName, ss] : exportReferencesGraph) {
StorePathSet storePaths;
for (auto & storePathS : ss) {
if (!store.isInStore(storePathS))
throw BuildError(
BuildResult::Failure::InputRejected,
"'exportReferencesGraph' contains a non-store path '%1%'",
storePathS);
storePaths.insert(store.toStorePath(storePathS).first);
}
res.insert_or_assign(fileName, storePaths);
}
return res;
}
StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const
{ {
// FIXME: cache this? // FIXME: cache this?
StringSet res; StringSet res;
@@ -393,8 +318,7 @@ StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivat
return res; return res;
} }
template<typename Input> bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
{ {
if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform) if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform)
&& !drv.isBuiltin()) && !drv.isBuiltin())
@@ -410,194 +334,42 @@ bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDe
return true; return true;
} }
template<typename Input> bool DerivationOptions::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
bool DerivationOptions<Input>::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
{ {
return preferLocalBuild && canBuildLocally(localStore, drv); return preferLocalBuild && canBuildLocally(localStore, drv);
} }
template<typename Input> bool DerivationOptions::substitutesAllowed() const
bool DerivationOptions<Input>::substitutesAllowed() const
{ {
return settings.alwaysAllowSubstitutes ? true : allowSubstitutes; return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
} }
template<typename Input> bool DerivationOptions::useUidRange(const BasicDerivation & drv) const
bool DerivationOptions<Input>::useUidRange(const BasicDerivation & drv) const
{ {
return getRequiredSystemFeatures(drv).count("uid-range"); return getRequiredSystemFeatures(drv).count("uid-range");
} }
std::optional<DerivationOptions<StorePath>> tryResolve(
const DerivationOptions<SingleDerivedPath> & drvOptions,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain)
{
auto tryResolvePath = [&](const SingleDerivedPath & input) -> std::optional<StorePath> {
return std::visit(
overloaded{
[](const SingleDerivedPath::Opaque & p) -> std::optional<StorePath> { return p.path; },
[&](const SingleDerivedPath::Built & p) -> std::optional<StorePath> {
return queryResolutionChain(p.drvPath, p.output);
}},
input.raw());
};
auto tryResolveRef = [&](const DrvRef<SingleDerivedPath> & ref) -> std::optional<DrvRef<StorePath>> {
return std::visit(
overloaded{
[](const OutputName & outputName) -> std::optional<DrvRef<StorePath>> { return outputName; },
[&](const SingleDerivedPath & input) -> std::optional<DrvRef<StorePath>> {
return tryResolvePath(input);
}},
ref);
};
auto tryResolveRefSet =
[&](const std::set<DrvRef<SingleDerivedPath>> & refSet) -> std::optional<std::set<DrvRef<StorePath>>> {
std::set<DrvRef<StorePath>> resolvedSet;
for (const auto & ref : refSet) {
auto resolvedRef = tryResolveRef(ref);
if (!resolvedRef)
return std::nullopt;
resolvedSet.insert(*resolvedRef);
}
return resolvedSet;
};
// Helper function to try resolving OutputChecks using functional style
auto tryResolveOutputChecks = [&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
-> std::optional<DerivationOptions<StorePath>::OutputChecks> {
std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedReferences;
if (checks.allowedReferences) {
resolvedAllowedReferences = tryResolveRefSet(*checks.allowedReferences);
if (!resolvedAllowedReferences)
return std::nullopt;
}
std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedRequisites;
if (checks.allowedRequisites) {
resolvedAllowedRequisites = tryResolveRefSet(*checks.allowedRequisites);
if (!resolvedAllowedRequisites)
return std::nullopt;
}
auto resolvedDisallowedReferences = tryResolveRefSet(checks.disallowedReferences);
if (!resolvedDisallowedReferences)
return std::nullopt;
auto resolvedDisallowedRequisites = tryResolveRefSet(checks.disallowedRequisites);
if (!resolvedDisallowedRequisites)
return std::nullopt;
return DerivationOptions<StorePath>::OutputChecks{
.ignoreSelfRefs = checks.ignoreSelfRefs,
.maxSize = checks.maxSize,
.maxClosureSize = checks.maxClosureSize,
.allowedReferences = resolvedAllowedReferences,
.disallowedReferences = *resolvedDisallowedReferences,
.allowedRequisites = resolvedAllowedRequisites,
.disallowedRequisites = *resolvedDisallowedRequisites,
};
};
// Helper function to resolve exportReferencesGraph using functional style
auto tryResolveExportReferencesGraph = [&](const std::map<std::string, std::set<SingleDerivedPath>> & exportGraph)
-> std::optional<std::map<std::string, std::set<StorePath>>> {
std::map<std::string, std::set<StorePath>> resolved;
for (const auto & [name, inputPaths] : exportGraph) {
std::set<StorePath> resolvedPaths;
for (const auto & inputPath : inputPaths) {
auto resolvedPath = tryResolvePath(inputPath);
if (!resolvedPath)
return std::nullopt;
resolvedPaths.insert(*resolvedPath);
}
resolved.emplace(name, std::move(resolvedPaths));
}
return resolved;
};
// Resolve outputChecks using functional style with std::visit
auto resolvedOutputChecks = std::visit(
overloaded{
[&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
-> std::optional<std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
auto resolved = tryResolveOutputChecks(checks);
if (!resolved)
return std::nullopt;
return std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(*resolved);
},
[&](const std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks> & checksMap)
-> std::optional<std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
std::map<std::string, DerivationOptions<StorePath>::OutputChecks> resolvedMap;
for (const auto & [outputName, checks] : checksMap) {
auto resolved = tryResolveOutputChecks(checks);
if (!resolved)
return std::nullopt;
resolvedMap.emplace(outputName, *resolved);
}
return std::variant<
DerivationOptions<StorePath>::OutputChecks,
std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(resolvedMap);
}},
drvOptions.outputChecks);
if (!resolvedOutputChecks)
return std::nullopt;
// Resolve exportReferencesGraph
auto resolvedExportGraph = tryResolveExportReferencesGraph(drvOptions.exportReferencesGraph);
if (!resolvedExportGraph)
return std::nullopt;
// Return resolved DerivationOptions using designated initializers
return DerivationOptions<StorePath>{
.outputChecks = *resolvedOutputChecks,
.unsafeDiscardReferences = drvOptions.unsafeDiscardReferences,
.passAsFile = drvOptions.passAsFile,
.exportReferencesGraph = *resolvedExportGraph,
.additionalSandboxProfile = drvOptions.additionalSandboxProfile,
.noChroot = drvOptions.noChroot,
.impureHostDeps = drvOptions.impureHostDeps,
.impureEnvVars = drvOptions.impureEnvVars,
.allowLocalNetworking = drvOptions.allowLocalNetworking,
.requiredSystemFeatures = drvOptions.requiredSystemFeatures,
.preferLocalBuild = drvOptions.preferLocalBuild,
.allowSubstitutes = drvOptions.allowSubstitutes,
};
}
template struct DerivationOptions<StorePath>;
template struct DerivationOptions<SingleDerivedPath>;
} // namespace nix } // namespace nix
namespace nlohmann { namespace nlohmann {
using namespace nix; using namespace nix;
DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDerivedPath>>::from_json(const json & json_) DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json_)
{ {
auto & json = getObject(json_); auto & json = getObject(json_);
return { return {
.outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> { .outputChecks = [&]() -> OutputChecksVariant {
auto outputChecks = getObject(valueAt(json, "outputChecks")); auto outputChecks = getObject(valueAt(json, "outputChecks"));
auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs"); auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs");
auto perOutputOpt = optionalValueAt(outputChecks, "perOutput"); auto perOutputOpt = optionalValueAt(outputChecks, "perOutput");
if (forAllOutputsOpt && !perOutputOpt) { if (forAllOutputsOpt && !perOutputOpt) {
return static_cast<OutputChecks<SingleDerivedPath>>(*forAllOutputsOpt); return static_cast<OutputChecks>(*forAllOutputsOpt);
} else if (perOutputOpt && !forAllOutputsOpt) { } else if (perOutputOpt && !forAllOutputsOpt) {
return static_cast<std::map<std::string, OutputChecks<SingleDerivedPath>>>(*perOutputOpt); return static_cast<std::map<std::string, OutputChecks>>(*perOutputOpt);
} else { } else {
throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required"); throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required");
} }
@@ -605,7 +377,7 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
.unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"), .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"),
.passAsFile = getStringSet(valueAt(json, "passAsFile")), .passAsFile = getStringSet(valueAt(json, "passAsFile")),
.exportReferencesGraph = valueAt(json, "exportReferencesGraph"), .exportReferencesGraph = getMap<StringSet>(getObject(valueAt(json, "exportReferencesGraph")), getStringSet),
.additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")), .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")),
.noChroot = getBoolean(valueAt(json, "noChroot")), .noChroot = getBoolean(valueAt(json, "noChroot")),
@@ -619,17 +391,16 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
}; };
} }
void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json( void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOptions & o)
json & json, const DerivationOptions<SingleDerivedPath> & o)
{ {
json["outputChecks"] = std::visit( json["outputChecks"] = std::visit(
overloaded{ overloaded{
[&](const OutputChecks<SingleDerivedPath> & checks) { [&](const OutputChecks & checks) {
nlohmann::json outputChecks; nlohmann::json outputChecks;
outputChecks["forAllOutputs"] = checks; outputChecks["forAllOutputs"] = checks;
return outputChecks; return outputChecks;
}, },
[&](const std::map<std::string, OutputChecks<SingleDerivedPath>> & checksPerOutput) { [&](const std::map<std::string, OutputChecks> & checksPerOutput) {
nlohmann::json outputChecks; nlohmann::json outputChecks;
outputChecks["perOutput"] = checksPerOutput; outputChecks["perOutput"] = checksPerOutput;
return outputChecks; return outputChecks;
@@ -652,7 +423,7 @@ void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json(
json["allowSubstitutes"] = o.allowSubstitutes; json["allowSubstitutes"] = o.allowSubstitutes;
} }
OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>::from_json(const json & json_) DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json_)
{ {
auto & json = getObject(json_); auto & json = getObject(json_);
@@ -660,16 +431,14 @@ OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>:
.ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")), .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
.maxSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxSize"))), .maxSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxSize"))),
.maxClosureSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxClosureSize"))), .maxClosureSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxClosureSize"))),
.allowedReferences = .allowedReferences = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedReferences"))),
ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedReferences"))), .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
.disallowedReferences = valueAt(json, "disallowedReferences"), .allowedRequisites = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedRequisites"))),
.allowedRequisites = .disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedRequisites"))),
.disallowedRequisites = valueAt(json, "disallowedRequisites"),
}; };
} }
void adl_serializer<OutputChecks<SingleDerivedPath>>::to_json(json & json, const OutputChecks<SingleDerivedPath> & c) void adl_serializer<DerivationOptions::OutputChecks>::to_json(json & json, const DerivationOptions::OutputChecks & c)
{ {
json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["ignoreSelfRefs"] = c.ignoreSelfRefs;
json["maxSize"] = c.maxSize; json["maxSize"] = c.maxSize;
+51 -44
View File
@@ -13,7 +13,6 @@
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
#include <boost/unordered/concurrent_flat_map.hpp> #include <boost/unordered/concurrent_flat_map.hpp>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <optional>
namespace nix { namespace nix {
@@ -792,63 +791,71 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName)
DerivationType BasicDerivation::type() const DerivationType BasicDerivation::type() const
{ {
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs,
impureOutputs;
std::optional<HashAlgorithm> floatingHashAlgo; std::optional<HashAlgorithm> floatingHashAlgo;
std::optional<DerivationType> ty;
auto decide = [&](DerivationType newTy) {
if (!ty)
ty = newTy;
else if (ty.value() != newTy)
throw Error("can't mix derivation output types");
else if (ty.value() == DerivationType::ContentAddressed{.sandboxed = false, .fixed = true})
// FIXME: Experimental feature?
throw Error("only one fixed output is allowed for now");
};
for (auto & i : outputs) { for (auto & i : outputs) {
std::visit( std::visit(
overloaded{ overloaded{
[&](const DerivationOutput::InputAddressed &) { [&](const DerivationOutput::InputAddressed &) { inputAddressedOutputs.insert(i.first); },
decide( [&](const DerivationOutput::CAFixed &) { fixedCAOutputs.insert(i.first); },
DerivationType::InputAddressed{
.deferred = false,
});
},
[&](const DerivationOutput::CAFixed &) {
decide(
DerivationType::ContentAddressed{
.sandboxed = false,
.fixed = true,
});
if (i.first != "out"sv)
throw Error("single fixed output must be named \"out\"");
},
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
decide( floatingCAOutputs.insert(i.first);
DerivationType::ContentAddressed{ if (!floatingHashAlgo) {
.sandboxed = true,
.fixed = false,
});
if (!floatingHashAlgo)
floatingHashAlgo = dof.hashAlgo; floatingHashAlgo = dof.hashAlgo;
else if (*floatingHashAlgo != dof.hashAlgo) } else {
throw Error("all floating outputs must use the same hash algorithm"); if (*floatingHashAlgo != dof.hashAlgo)
throw Error("all floating outputs must use the same hash algorithm");
}
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) { deferredIAOutputs.insert(i.first); },
decide( [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); },
DerivationType::InputAddressed{
.deferred = true,
});
},
[&](const DerivationOutput::Impure &) { decide(DerivationType::Impure{}); },
}, },
i.second.raw); i.second.raw);
} }
if (!ty) if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty())
throw Error("must have at least one output"); throw Error("must have at least one output");
return ty.value(); if (!inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty())
return DerivationType::InputAddressed{
.deferred = false,
};
if (inputAddressedOutputs.empty() && !fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out"sv)
throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed{
.sandboxed = false,
.fixed = true,
};
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && !floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && impureOutputs.empty())
return DerivationType::ContentAddressed{
.sandboxed = true,
.fixed = false,
};
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& !deferredIAOutputs.empty() && impureOutputs.empty())
return DerivationType::InputAddressed{
.deferred = true,
};
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
&& deferredIAOutputs.empty() && !impureOutputs.empty())
return DerivationType::Impure{};
throw Error("can't mix derivation output types");
} }
DrvHashes drvHashes; DrvHashes drvHashes;
-43
View File
@@ -1,6 +1,5 @@
#include "nix/store/downstream-placeholder.hh" #include "nix/store/downstream-placeholder.hh"
#include "nix/store/derivations.hh" #include "nix/store/derivations.hh"
#include "nix/util/json-utils.hh"
namespace nix { namespace nix {
@@ -50,45 +49,3 @@ DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
} }
} // namespace nix } // namespace nix
namespace nlohmann {
using namespace nix;
template<typename Item>
DrvRef<Item> adl_serializer<DrvRef<Item>>::from_json(const json & json)
{
// OutputName case: { "drvPath": "self", "output": <output> }
if (json.type() == nlohmann::json::value_t::object) {
auto & obj = getObject(json);
if (auto * drvPath_ = get(obj, "drvPath")) {
auto & drvPath = *drvPath_;
if (drvPath.type() == nlohmann::json::value_t::string && getString(drvPath) == "self") {
return getString(valueAt(obj, "output"));
}
}
}
// Input case
return adl_serializer<Item>::from_json(json);
}
template<typename Item>
void adl_serializer<DrvRef<Item>>::to_json(json & json, const DrvRef<Item> & ref)
{
std::visit(
overloaded{
[&](const OutputName & outputName) {
json = nlohmann::json::object();
json["drvPath"] = "self";
json["output"] = outputName;
},
[&](const Item & item) { json = item; },
},
ref);
}
template struct adl_serializer<nix::DrvRef<StorePath>>;
template struct adl_serializer<nix::DrvRef<SingleDerivedPath>>;
} // namespace nlohmann
+69 -154
View File
@@ -48,7 +48,7 @@ struct curlFileTransfer : public FileTransfer
std::random_device rd; std::random_device rd;
std::mt19937 mt19937; std::mt19937 mt19937;
struct TransferItem : public std::enable_shared_from_this<TransferItem>, public FileTransfer::Item struct TransferItem : public std::enable_shared_from_this<TransferItem>
{ {
curlFileTransfer & fileTransfer; curlFileTransfer & fileTransfer;
FileTransferRequest request; FileTransferRequest request;
@@ -60,7 +60,6 @@ struct curlFileTransfer : public FileTransfer
// buffer to accompany the `req` above // buffer to accompany the `req` above
char errbuf[CURL_ERROR_SIZE]; char errbuf[CURL_ERROR_SIZE];
bool active = false; // whether the handle has been added to the multi object bool active = false; // whether the handle has been added to the multi object
bool paused = false; // whether the request has been paused previously
std::string statusMsg; std::string statusMsg;
unsigned int attempt = 0; unsigned int attempt = 0;
@@ -117,13 +116,7 @@ struct curlFileTransfer : public FileTransfer
successful response. */ successful response. */
if (successfulStatuses.count(httpStatus)) { if (successfulStatuses.count(httpStatus)) {
writtenToSink += data.size(); writtenToSink += data.size();
PauseTransfer needsPause = this->request.dataCallback(data); this->request.dataCallback(data);
if (needsPause == PauseTransfer::Yes) {
/* Smuggle the boolean flag into writeCallback. Note that
the finalSink might get called multiple times if there's
decompression going on. */
paused = true;
}
} }
} else } else
this->result.data.append(data); this->result.data.append(data);
@@ -158,23 +151,15 @@ struct curlFileTransfer : public FileTransfer
} }
} }
void failEx(std::exception_ptr ex) noexcept void failEx(std::exception_ptr ex)
{ {
assert(!done); assert(!done);
done = true; done = true;
try {
std::rethrow_exception(ex);
} catch (nix::Error & e) {
/* Add more context to the error message. */
e.addTrace({}, "during %s of '%s'", Uncolored(request.verb()), request.uri.to_string());
} catch (...) {
/* Can't add more context to the error. */
}
callback.rethrow(ex); callback.rethrow(ex);
} }
template<class T> template<class T>
void fail(T && e) noexcept void fail(T && e)
{ {
failEx(std::make_exception_ptr(std::forward<T>(e))); failEx(std::make_exception_ptr(std::forward<T>(e)));
} }
@@ -183,38 +168,32 @@ struct curlFileTransfer : public FileTransfer
std::shared_ptr<FinishSink> decompressionSink; std::shared_ptr<FinishSink> decompressionSink;
std::optional<StringSink> errorSink; std::optional<StringSink> errorSink;
std::exception_ptr callbackException; std::exception_ptr writeException;
size_t writeCallback(void * contents, size_t size, size_t nmemb) noexcept size_t writeCallback(void * contents, size_t size, size_t nmemb)
try { {
size_t realSize = size * nmemb; try {
result.bodySize += realSize; size_t realSize = size * nmemb;
result.bodySize += realSize;
if (!decompressionSink) { if (!decompressionSink) {
decompressionSink = makeDecompressionSink(encoding, finalSink); decompressionSink = makeDecompressionSink(encoding, finalSink);
if (!successfulStatuses.count(getHTTPStatus())) { if (!successfulStatuses.count(getHTTPStatus())) {
// In this case we want to construct a TeeSink, to keep // In this case we want to construct a TeeSink, to keep
// the response around (which we figure won't be big // the response around (which we figure won't be big
// like an actual download should be) to improve error // like an actual download should be) to improve error
// messages. // messages.
errorSink = StringSink{}; errorSink = StringSink{};
}
} }
}
(*decompressionSink)({(char *) contents, realSize}); (*decompressionSink)({(char *) contents, realSize});
if (paused) {
/* The callback has signaled that the transfer needs to be
paused. Already consumed data won't be returned twice unlike
when returning CURL_WRITEFUNC_PAUSE.
https://curl-library.cool.haxx.narkive.com/larE1cRA/curl-easy-pause-documentation-question
*/
curl_easy_pause(req, CURLPAUSE_RECV);
}
return realSize; return realSize;
} catch (...) { } catch (...) {
callbackException = std::current_exception(); writeException = std::current_exception();
return 0; return 0;
}
} }
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -230,8 +209,8 @@ struct curlFileTransfer : public FileTransfer
result.urls.push_back(effectiveUriCStr); result.urls.push_back(effectiveUriCStr);
} }
size_t headerCallback(void * contents, size_t size, size_t nmemb) noexcept size_t headerCallback(void * contents, size_t size, size_t nmemb)
try { {
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
std::string line((char *) contents, realSize); std::string line((char *) contents, realSize);
printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
@@ -284,15 +263,6 @@ struct curlFileTransfer : public FileTransfer
} }
} }
return realSize; return realSize;
} catch (...) {
#if LIBCURL_VERSION_NUM >= 0x075700
/* https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html:
You can also abort the transfer by returning CURL_WRITEFUNC_ERROR. */
callbackException = std::current_exception();
return CURL_WRITEFUNC_ERROR;
#else
return realSize;
#endif
} }
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -300,17 +270,14 @@ struct curlFileTransfer : public FileTransfer
return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
} }
int progressCallback(curl_off_t dltotal, curl_off_t dlnow) noexcept int progressCallback(curl_off_t dltotal, curl_off_t dlnow)
try { {
act.progress(dlnow, dltotal); try {
act.progress(dlnow, dltotal);
} catch (nix::Interrupted &) {
assert(getInterrupted());
}
return getInterrupted(); return getInterrupted();
} catch (nix::Interrupted &) {
assert(getInterrupted());
return 1;
} catch (...) {
/* Something unexpected has happened like logger throwing an exception. */
callbackException = std::current_exception();
return 1;
} }
static int progressCallbackWrapper( static int progressCallbackWrapper(
@@ -321,14 +288,11 @@ struct curlFileTransfer : public FileTransfer
return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow); return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow);
} }
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) noexcept static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
try { {
if (type == CURLINFO_TEXT) if (type == CURLINFO_TEXT)
vomit("curl: %s", chomp(std::string(data, size))); vomit("curl: %s", chomp(std::string(data, size)));
return 0; return 0;
} catch (...) {
/* Swallow the exception. Nothing left to do. */
return 0;
} }
size_t readCallback(char * buffer, size_t size, size_t nitems) noexcept size_t readCallback(char * buffer, size_t size, size_t nitems) noexcept
@@ -338,7 +302,6 @@ struct curlFileTransfer : public FileTransfer
} catch (EndOfFile &) { } catch (EndOfFile &) {
return 0; return 0;
} catch (...) { } catch (...) {
callbackException = std::current_exception();
return CURL_READFUNC_ABORT; return CURL_READFUNC_ABORT;
} }
@@ -370,7 +333,6 @@ struct curlFileTransfer : public FileTransfer
} }
return CURL_SEEKFUNC_OK; return CURL_SEEKFUNC_OK;
} catch (...) { } catch (...) {
callbackException = std::current_exception();
return CURL_SEEKFUNC_FAIL; return CURL_SEEKFUNC_FAIL;
} }
@@ -379,15 +341,6 @@ struct curlFileTransfer : public FileTransfer
return ((TransferItem *) clientp)->seekCallback(offset, origin); return ((TransferItem *) clientp)->seekCallback(offset, origin);
} }
void unpause()
{
/* Unpausing an already unpaused transfer is a no-op. */
if (paused) {
curl_easy_pause(req, CURLPAUSE_CONT);
paused = false;
}
}
void init() void init()
{ {
if (!req) if (!req)
@@ -433,17 +386,17 @@ struct curlFileTransfer : public FileTransfer
if (settings.downloadSpeed.get() > 0) if (settings.downloadSpeed.get() > 0)
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
if (request.method == HttpMethod::Head) if (request.method == HttpMethod::HEAD)
curl_easy_setopt(req, CURLOPT_NOBODY, 1); curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (request.method == HttpMethod::Delete) if (request.method == HttpMethod::DELETE)
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
if (request.data) { if (request.data) {
if (request.method == HttpMethod::Post) { if (request.method == HttpMethod::POST) {
curl_easy_setopt(req, CURLOPT_POST, 1L); curl_easy_setopt(req, CURLOPT_POST, 1L);
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint); curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
} else if (request.method == HttpMethod::Put) { } else if (request.method == HttpMethod::PUT) {
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->sizeHint); curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->sizeHint);
} else { } else {
@@ -523,7 +476,7 @@ struct curlFileTransfer : public FileTransfer
try { try {
decompressionSink->finish(); decompressionSink->finish();
} catch (...) { } catch (...) {
callbackException = std::current_exception(); writeException = std::current_exception();
} }
} }
@@ -532,8 +485,8 @@ struct curlFileTransfer : public FileTransfer
httpStatus = 304; httpStatus = 304;
} }
if (callbackException) if (writeException)
failEx(callbackException); failEx(writeException);
else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
@@ -648,7 +601,7 @@ struct curlFileTransfer : public FileTransfer
errorSink.reset(); errorSink.reset();
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
try { try {
fileTransfer.enqueueItem(ref{shared_from_this()}); fileTransfer.enqueueItem(shared_from_this());
} catch (const nix::Error & e) { } catch (const nix::Error & e) {
// If enqueue fails (e.g., during shutdown), fail the transfer properly // If enqueue fails (e.g., during shutdown), fail the transfer properly
// instead of letting the exception propagate, which would leave done=false // instead of letting the exception propagate, which would leave done=false
@@ -665,24 +618,24 @@ struct curlFileTransfer : public FileTransfer
{ {
struct EmbargoComparator struct EmbargoComparator
{ {
bool operator()(const ref<TransferItem> & i1, const ref<TransferItem> & i2) bool operator()(const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2)
{ {
return i1->embargo > i2->embargo; return i1->embargo > i2->embargo;
} }
}; };
std::priority_queue<ref<TransferItem>, std::vector<ref<TransferItem>>, EmbargoComparator> incoming; std::
std::vector<ref<TransferItem>> unpause; priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator>
incoming;
private: private:
bool quitting = false; bool quitting = false;
public: public:
void quit() void quit()
{ {
quitting = true; quitting = true;
/* We will not be processing any more incoming requests */ /* We wil not be processing any more incoming requests */
while (!incoming.empty()) while (!incoming.empty())
incoming.pop(); incoming.pop();
unpause.clear();
} }
bool isQuitting() bool isQuitting()
@@ -851,17 +804,6 @@ struct curlFileTransfer : public FileTransfer
item->active = true; item->active = true;
items[item->req] = item; items[item->req] = item;
} }
/* NOTE: Unpausing may invoke callbacks to flush all buffers. */
auto unpause = [&]() {
auto state(state_.lock());
auto res = state->unpause;
state->unpause.clear();
return res;
}();
for (auto & item : unpause)
item->unpause();
} }
debug("download thread shutting down"); debug("download thread shutting down");
@@ -886,7 +828,7 @@ struct curlFileTransfer : public FileTransfer
} }
} }
ItemHandle enqueueItem(ref<TransferItem> item) void enqueueItem(std::shared_ptr<TransferItem> item)
{ {
if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https" if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
&& item->request.uri.scheme() != "s3") && item->request.uri.scheme() != "s3")
@@ -901,34 +843,19 @@ struct curlFileTransfer : public FileTransfer
#ifndef _WIN32 // TODO need graceful async exit support on Windows? #ifndef _WIN32 // TODO need graceful async exit support on Windows?
writeFull(wakeupPipe.writeSide.get(), " "); writeFull(wakeupPipe.writeSide.get(), " ");
#endif #endif
return ItemHandle(static_cast<Item &>(*item));
} }
ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
{ {
/* Handle s3:// URIs by converting to HTTPS and optionally adding auth */ /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */
if (request.uri.scheme() == "s3") { if (request.uri.scheme() == "s3") {
auto modifiedRequest = request; auto modifiedRequest = request;
modifiedRequest.setupForS3(); modifiedRequest.setupForS3();
return enqueueItem(make_ref<TransferItem>(*this, std::move(modifiedRequest), std::move(callback))); enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
return;
} }
return enqueueItem(make_ref<TransferItem>(*this, request, std::move(callback))); enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
}
void unpauseTransfer(ref<TransferItem> item)
{
auto state(state_.lock());
state->unpause.push_back(std::move(item));
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
writeFull(wakeupPipe.writeSide.get(), " ");
#endif
}
void unpauseTransfer(ItemHandle handle) override
{
unpauseTransfer(ref{static_cast<TransferItem &>(handle.item.get()).shared_from_this()});
} }
}; };
@@ -1025,7 +952,6 @@ void FileTransfer::download(
struct State struct State
{ {
bool quit = false; bool quit = false;
bool paused = false;
std::exception_ptr exc; std::exception_ptr exc;
std::string data; std::string data;
std::condition_variable avail, request; std::condition_variable avail, request;
@@ -1041,38 +967,31 @@ void FileTransfer::download(
state->request.notify_one(); state->request.notify_one();
}); });
request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer { request.dataCallback = [_state](std::string_view data) {
auto state(_state->lock()); auto state(_state->lock());
if (state->quit) if (state->quit)
return PauseTransfer::No; return;
/* If the buffer is full, then go to sleep until the calling
thread wakes us up (i.e. when it has removed data from the
buffer). We don't wait forever to prevent stalling the
download thread. (Hopefully sleeping will throttle the
sender.) */
if (state->data.size() > fileTransferSettings.downloadBufferSize) {
debug("download buffer is full; going to sleep");
static bool haveWarned = false;
warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
state.wait_for(state->request, std::chrono::seconds(10));
}
/* Append data to the buffer and wake up the calling /* Append data to the buffer and wake up the calling
thread. */ thread. */
state->data.append(data); state->data.append(data);
state->avail.notify_one(); state->avail.notify_one();
if (state->data.size() <= fileTransferSettings.downloadBufferSize)
return PauseTransfer::No;
/* dataCallback gets called multiple times by an intermediate sink. Only
issue the debug message the first time around. */
if (!state->paused)
debug(
"pausing transfer for '%s': download buffer is full (%d > %d)",
uri,
state->data.size(),
fileTransferSettings.downloadBufferSize);
state->paused = true;
/* Technically the buffer might become larger than
downloadBufferSize, but with sinks there's no way to avoid
consuming data. */
return PauseTransfer::Yes;
}; };
auto handle = enqueueFileTransfer( enqueueFileTransfer(
request, {[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) { request, {[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) {
auto state(_state->lock()); auto state(_state->lock());
state->quit = true; state->quit = true;
@@ -1105,10 +1024,6 @@ void FileTransfer::download(
return; return;
} }
if (state->paused) {
unpauseTransfer(handle);
state->paused = false;
}
state.wait(state->avail); state.wait(state->avail);
if (state->data.empty()) if (state->data.empty())
+2 -2
View File
@@ -120,7 +120,7 @@ bool HttpBinaryCacheStore::fileExists(const std::string & path)
try { try {
FileTransferRequest request(makeRequest(path)); FileTransferRequest request(makeRequest(path));
request.method = HttpMethod::Head; request.method = HttpMethod::HEAD;
getFileTransfer()->download(request); getFileTransfer()->download(request);
return true; return true;
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
@@ -141,7 +141,7 @@ void HttpBinaryCacheStore::upload(
std::optional<Headers> headers) std::optional<Headers> headers)
{ {
auto req = makeRequest(path); auto req = makeRequest(path);
req.method = HttpMethod::Put; req.method = HttpMethod::PUT;
if (headers) { if (headers) {
req.headers.reserve(req.headers.size() + headers->size()); req.headers.reserve(req.headers.size() + headers->size());
@@ -69,7 +69,7 @@ struct DerivationBuilderParams
* *
* @todo this should be part of `Derivation`. * @todo this should be part of `Derivation`.
*/ */
const DerivationOptions<StorePath> & drvOptions; const DerivationOptions & drvOptions;
// The remainder is state held during the build. // The remainder is state held during the build.
@@ -52,6 +52,8 @@ private:
*/ */
std::unique_ptr<Derivation> drv; std::unique_ptr<Derivation> drv;
std::unique_ptr<DerivationOptions> drvOptions;
/** /**
* The remainder is state held during the build. * The remainder is state held during the build.
*/ */
@@ -113,8 +115,7 @@ private:
/** /**
* Is the build hook willing to perform the build? * Is the build hook willing to perform the build?
*/ */
HookReply tryBuildHook( HookReply tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs);
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions);
/** /**
* Open a log file and a pipe to it. * Open a log file and a pipe to it.
@@ -8,7 +8,6 @@ namespace nix {
class Store; class Store;
struct Derivation; struct Derivation;
template<typename Input>
struct DerivationOptions; struct DerivationOptions;
/** /**
@@ -78,10 +77,7 @@ struct DesugaredEnv
* just part of `Derivation`. * just part of `Derivation`.
*/ */
static DesugaredEnv create( static DesugaredEnv create(
Store & store, Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths);
const Derivation & drv,
const DerivationOptions<StorePath> & drvOptions,
const StorePathSet & inputPaths);
}; };
} // namespace nix } // namespace nix
@@ -8,8 +8,7 @@
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/json-impls.hh" #include "nix/util/json-impls.hh"
#include "nix/store/store-dir-config.hh" #include "nix/store/path.hh"
#include "nix/store/downstream-placeholder.hh"
namespace nix { namespace nix {
@@ -18,9 +17,6 @@ struct StoreDirConfig;
struct BasicDerivation; struct BasicDerivation;
struct StructuredAttrs; struct StructuredAttrs;
template<typename V>
struct DerivedPathMap;
/** /**
* This represents all the special options on a `Derivation`. * This represents all the special options on a `Derivation`.
* *
@@ -38,7 +34,6 @@ struct DerivedPathMap;
* separately. That would be nice to separate concerns, and not make any * separately. That would be nice to separate concerns, and not make any
* environment variable names magical. * environment variable names magical.
*/ */
template<typename Input>
struct DerivationOptions struct DerivationOptions
{ {
struct OutputChecks struct OutputChecks
@@ -46,15 +41,13 @@ struct DerivationOptions
bool ignoreSelfRefs = false; bool ignoreSelfRefs = false;
std::optional<uint64_t> maxSize, maxClosureSize; std::optional<uint64_t> maxSize, maxClosureSize;
using DrvRef = nix::DrvRef<Input>;
/** /**
* env: allowedReferences * env: allowedReferences
* *
* A value of `nullopt` indicates that the check is skipped. * A value of `nullopt` indicates that the check is skipped.
* This means that all references are allowed. * This means that all references are allowed.
*/ */
std::optional<std::set<DrvRef>> allowedReferences; std::optional<StringSet> allowedReferences;
/** /**
* env: disallowedReferences * env: disallowedReferences
@@ -62,21 +55,21 @@ struct DerivationOptions
* No needed for `std::optional`, because skipping the check is * No needed for `std::optional`, because skipping the check is
* the same as disallowing the references. * the same as disallowing the references.
*/ */
std::set<DrvRef> disallowedReferences; StringSet disallowedReferences;
/** /**
* env: allowedRequisites * env: allowedRequisites
* *
* See `allowedReferences` * See `allowedReferences`
*/ */
std::optional<std::set<DrvRef>> allowedRequisites; std::optional<StringSet> allowedRequisites;
/** /**
* env: disallowedRequisites * env: disallowedRequisites
* *
* See `disallowedReferences` * See `disallowedReferences`
*/ */
std::set<DrvRef> disallowedRequisites; StringSet disallowedRequisites;
bool operator==(const OutputChecks &) const = default; bool operator==(const OutputChecks &) const = default;
}; };
@@ -123,7 +116,23 @@ struct DerivationOptions
* attributes give to the builder. The set of paths in the original JSON * attributes give to the builder. The set of paths in the original JSON
* is replaced with a list of `PathInfo` in JSON format. * is replaced with a list of `PathInfo` in JSON format.
*/ */
std::map<std::string, std::set<Input>> exportReferencesGraph; std::map<std::string, StringSet> exportReferencesGraph;
/**
* Once a derivations is resolved, the strings in in
* `exportReferencesGraph` should all be store paths (with possible
* suffix paths, but those are discarded).
*
* @return The parsed path set for for each key in the map.
*
* @todo Ideally, `exportReferencesGraph` would just store
* `StorePath`s for this, but we can't just do that, because for CA
* derivations they is actually in general `DerivedPath`s (via
* placeholder strings) until the derivation is resolved and exact
* inputs store paths are known. We can use better types for that
* too, but that is a longer project.
*/
std::map<std::string, StorePathSet> getParsedExportReferencesGraph(const StoreDirConfig & store) const;
/** /**
* env: __sandboxProfile * env: __sandboxProfile
@@ -176,6 +185,18 @@ struct DerivationOptions
bool operator==(const DerivationOptions &) const = default; bool operator==(const DerivationOptions &) const = default;
/**
* Parse this information from its legacy encoding as part of the
* environment. This should not be used with nice greenfield formats
* (e.g. JSON) but is necessary for supporting old formats (e.g.
* ATerm).
*/
static DerivationOptions
fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
static DerivationOptions
fromStructuredAttrs(const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn = true);
/** /**
* @param drv Must be the same derivation we parsed this from. In * @param drv Must be the same derivation we parsed this from. In
* the future we'll flip things around so a `BasicDerivation` has * the future we'll flip things around so a `BasicDerivation` has
@@ -201,49 +222,7 @@ struct DerivationOptions
bool useUidRange(const BasicDerivation & drv) const; bool useUidRange(const BasicDerivation & drv) const;
}; };
extern template struct DerivationOptions<StorePath>;
extern template struct DerivationOptions<SingleDerivedPath>;
struct DerivationOutput;
/**
* Parse this information from its legacy encoding as part of the
* environment. This should not be used with nice greenfield formats
* (e.g. JSON) but is necessary for supporting old formats (e.g.
* ATerm).
*/
DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
const StoreDirConfig & store,
const DerivedPathMap<StringSet> & inputDrvs,
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn = true,
const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs(
const StoreDirConfig & store,
const StringMap & env,
const StructuredAttrs * parsed,
bool shouldWarn = true,
const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
/**
* This is the counterpart of `Derivation::tryResolve`. In particular,
* it takes the same sort of callback, which is used to reolve
* non-constant deriving paths.
*
* We need this function when resolving a derivation, and we will use
* this as part of that if/when `Derivation` includes
* `DerivationOptions`
*/
std::optional<DerivationOptions<StorePath>> tryResolve(
const DerivationOptions<SingleDerivedPath> & drvOptions,
std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
queryResolutionChain);
}; // namespace nix }; // namespace nix
JSON_IMPL(nix::DerivationOptions<nix::StorePath>); JSON_IMPL(DerivationOptions);
JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>); JSON_IMPL(DerivationOptions::OutputChecks)
JSON_IMPL(nix::DerivationOptions<nix::StorePath>::OutputChecks)
JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>::OutputChecks)
@@ -2,23 +2,11 @@
///@file ///@file
#include "nix/util/hash.hh" #include "nix/util/hash.hh"
#include "nix/util/json-impls.hh"
#include "nix/store/path.hh" #include "nix/store/path.hh"
#include "nix/store/derived-path.hh" #include "nix/store/derived-path.hh"
namespace nix { namespace nix {
/**
* A reference is either to a to-be-registered output (by name),
* or to an already-registered store object (by `Input`).
*
* `Ref<SingleDerivedPath` is a representation of something that can be
* turned into a placeholder. (Regular own-output placeholder in the
* first case, `DownstreamPlaceholder` in the second case.)
*/
template<typename Input>
using DrvRef = std::variant<OutputName, Input>;
/** /**
* Downstream Placeholders are opaque and almost certainly unique values * Downstream Placeholders are opaque and almost certainly unique values
* used to allow derivations to refer to store objects which are yet to * used to allow derivations to refer to store objects which are yet to
@@ -104,17 +92,3 @@ public:
}; };
} // namespace nix } // namespace nix
namespace nlohmann {
template<typename Item>
struct adl_serializer<nix::DrvRef<Item>>
{
static nix::DrvRef<Item> from_json(const json & json);
static void to_json(json & json, const nix::DrvRef<Item> & t);
};
extern template struct adl_serializer<nix::DrvRef<nix::StorePath>>;
extern template struct adl_serializer<nix::DrvRef<nix::SingleDerivedPath>>;
} // namespace nlohmann
+15 -52
View File
@@ -70,12 +70,12 @@ struct FileTransferSettings : Config
Setting<size_t> downloadBufferSize{ Setting<size_t> downloadBufferSize{
this, this,
1 * 1024 * 1024, 64 * 1024 * 1024,
"download-buffer-size", "download-buffer-size",
R"( R"(
The size of Nix's internal download buffer in bytes during `curl` transfers. If data is The size of Nix's internal download buffer in bytes during `curl` transfers. If data is
not processed quickly enough to exceed the size of this buffer, downloads may stall. not processed quickly enough to exceed the size of this buffer, downloads may stall.
The default is 1048576 (1 MiB). The default is 67108864 (64 MiB).
)"}; )"};
}; };
@@ -87,11 +87,11 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT;
* HTTP methods supported by FileTransfer. * HTTP methods supported by FileTransfer.
*/ */
enum struct HttpMethod { enum struct HttpMethod {
Get, GET,
Put, PUT,
Head, HEAD,
Post, POST,
Delete, DELETE,
}; };
/** /**
@@ -105,17 +105,12 @@ struct UsernameAuth
std::optional<std::string> password; std::optional<std::string> password;
}; };
enum class PauseTransfer : bool {
No = false,
Yes = true,
};
struct FileTransferRequest struct FileTransferRequest
{ {
VerbatimURL uri; VerbatimURL uri;
Headers headers; Headers headers;
std::string expectedETag; std::string expectedETag;
HttpMethod method = HttpMethod::Get; HttpMethod method = HttpMethod::GET;
size_t tries = fileTransferSettings.tries; size_t tries = fileTransferSettings.tries;
unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
ActivityId parentAct; ActivityId parentAct;
@@ -141,14 +136,7 @@ struct FileTransferRequest
std::optional<UploadData> data; std::optional<UploadData> data;
std::string mimeType; std::string mimeType;
std::function<void(std::string_view data)> dataCallback;
/**
* Callbacked invoked with a chunk of received data.
* Can pause the transfer by returning PauseTransfer::Yes. No data must be consumed
* if transfer is paused.
*/
std::function<PauseTransfer(std::string_view data)> dataCallback;
/** /**
* Optional username and password for HTTP basic authentication. * Optional username and password for HTTP basic authentication.
* When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option. * When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option.
@@ -176,14 +164,14 @@ struct FileTransferRequest
std::string verb() const std::string verb() const
{ {
switch (method) { switch (method) {
case HttpMethod::Head: case HttpMethod::HEAD:
case HttpMethod::Get: case HttpMethod::GET:
return "download"; return "download";
case HttpMethod::Put: case HttpMethod::PUT:
case HttpMethod::Post: case HttpMethod::POST:
assert(data); assert(data);
return "upload"; return "upload";
case HttpMethod::Delete: case HttpMethod::DELETE:
return "delet"; return "delet";
} }
unreachable(); unreachable();
@@ -238,25 +226,6 @@ class Store;
struct FileTransfer struct FileTransfer
{ {
protected:
class Item
{};
public:
/**
* An opaque handle to the file transfer. Can be used to reference an in-flight transfer operations.
*/
struct ItemHandle
{
std::reference_wrapper<Item> item;
friend struct FileTransfer;
ItemHandle(Item & item)
: item(item)
{
}
};
virtual ~FileTransfer() {} virtual ~FileTransfer() {}
/** /**
@@ -264,13 +233,7 @@ public:
* the download. The future may throw a FileTransferError * the download. The future may throw a FileTransferError
* exception. * exception.
*/ */
virtual ItemHandle virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
/**
* Unpause a transfer that has been previously paused by a dataCallback.
*/
virtual void unpauseTransfer(ItemHandle handle) = 0;
std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request); std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
@@ -55,6 +55,7 @@ headers = [ config_pub_h ] + files(
'machines.hh', 'machines.hh',
'make-content-addressed.hh', 'make-content-addressed.hh',
'names.hh', 'names.hh',
'nar-accessor.hh',
'nar-info-disk-cache.hh', 'nar-info-disk-cache.hh',
'nar-info.hh', 'nar-info.hh',
'outputs-spec.hh', 'outputs-spec.hh',
@@ -0,0 +1,43 @@
#pragma once
///@file
#include "nix/util/source-accessor.hh"
#include <functional>
#include <nlohmann/json_fwd.hpp>
namespace nix {
struct Source;
/**
* Return an object that provides access to the contents of a NAR
* file.
*/
ref<SourceAccessor> makeNarAccessor(std::string && nar);
ref<SourceAccessor> makeNarAccessor(Source & source);
/**
* Create a NAR accessor from a NAR listing (in the format produced by
* listNar()). The callback getNarBytes(offset, length) is used by the
* readFile() method of the accessor to get the contents of files
* inside the NAR.
*/
using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
/**
* The canonical GetNarBytes function for a seekable Source.
*/
GetNarBytes seekableGetNarBytes(const Path & path);
ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
/**
* Write a JSON representation of the contents of a NAR (except file
* contents).
*/
nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
} // namespace nix
@@ -9,7 +9,6 @@
namespace nix { namespace nix {
class Store; class Store;
template<typename Input>
struct DerivationOptions; struct DerivationOptions;
struct DerivationOutput; struct DerivationOutput;
@@ -48,7 +47,7 @@ struct StructuredAttrs
nlohmann::json::object_t prepareStructuredAttrs( nlohmann::json::object_t prepareStructuredAttrs(
Store & store, Store & store,
const DerivationOptions<StorePath> & drvOptions, const DerivationOptions & drvOptions,
const StorePathSet & inputPaths, const StorePathSet & inputPaths,
const DerivationOutputs & outputs) const; const DerivationOutputs & outputs) const;
+9 -5
View File
@@ -1246,8 +1246,10 @@ StorePath LocalStore::addToStoreFromDump(
auto desc = ContentAddressWithReferences::fromParts( auto desc = ContentAddressWithReferences::fromParts(
hashMethod, hashMethod,
methodsMatch ? dumpHash methodsMatch
: hashPath(makeFSSourceAccessor(tempPath), hashMethod.getFileIngestionMethod(), hashAlgo).first, ? dumpHash
: hashPath(PosixSourceAccessor::createAtRoot(tempPath), hashMethod.getFileIngestionMethod(), hashAlgo)
.first,
{ {
.others = references, .others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1383,9 +1385,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
checkInterrupt(); checkInterrupt();
auto name = link.path().filename(); auto name = link.path().filename();
printMsg(lvlTalkative, "checking contents of %s", name); printMsg(lvlTalkative, "checking contents of %s", name);
std::string hash = std::string hash = hashPath(
hashPath(makeFSSourceAccessor(link.path()), FileIngestionMethod::NixArchive, HashAlgorithm::SHA256) PosixSourceAccessor::createAtRoot(link.path()),
.first.to_string(HashFormat::Nix32, false); FileIngestionMethod::NixArchive,
HashAlgorithm::SHA256)
.first.to_string(HashFormat::Nix32, false);
if (hash != name.string()) { if (hash != name.string()) {
printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash); printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash);
if (repair) { if (repair) {
+1 -11
View File
@@ -114,17 +114,6 @@ boost = dependency(
deps_other += boost deps_other += boost
curl = dependency('libcurl', 'curl', version : '>= 7.75.0') curl = dependency('libcurl', 'curl', version : '>= 7.75.0')
if curl.version().version_compare('>=8.16.0') and curl.version().version_compare(
'<8.17.0',
)
# Out of precaution, avoid building with libcurl version that suffer from https://github.com/curl/curl/issues/19334.
error(
'curl @0@ has issues with write pausing, please use libcurl < 8.16 or >= 8.17, see https://github.com/curl/curl/issues/19334'.format(
curl.version(),
),
)
endif
deps_private += curl deps_private += curl
# seccomp only makes sense on Linux # seccomp only makes sense on Linux
@@ -311,6 +300,7 @@ sources = files(
'make-content-addressed.cc', 'make-content-addressed.cc',
'misc.cc', 'misc.cc',
'names.cc', 'names.cc',
'nar-accessor.cc',
'nar-info-disk-cache.cc', 'nar-info-disk-cache.cc',
'nar-info.cc', 'nar-info.cc',
'optimise-store.cc', 'optimise-store.cc',
+2 -3
View File
@@ -225,12 +225,11 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
return; return;
auto drv = make_ref<Derivation>(derivationFromPath(drvPath)); auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
DerivationOptions<SingleDerivedPath> drvOptions; DerivationOptions drvOptions;
try { try {
// FIXME: this is a lot of work just to get the value // FIXME: this is a lot of work just to get the value
// of `allowSubstitutes`. // of `allowSubstitutes`.
drvOptions = derivationOptionsFromStructuredAttrs( drvOptions = DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
*this, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
throw; throw;
@@ -1,4 +1,4 @@
#include "nix/util/nar-accessor.hh" #include "nix/store/nar-accessor.hh"
#include "nix/util/archive.hh" #include "nix/util/archive.hh"
#include <map> #include <map>
@@ -272,39 +272,41 @@ GetNarBytes seekableGetNarBytes(const Path & path)
}; };
} }
template<bool deep> using nlohmann::json;
using ListNarResult = std::conditional_t<deep, NarListing, ShallowNarListing>;
template<bool deep> json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPath & path)
{ {
auto st = accessor.lstat(path); auto st = accessor->lstat(path);
json obj = json::object();
switch (st.type) { switch (st.type) {
case SourceAccessor::Type::tRegular: case SourceAccessor::Type::tRegular:
return typename ListNarResult<deep>::Regular{ obj["type"] = "regular";
.executable = st.isExecutable, if (st.fileSize)
.contents = obj["size"] = *st.fileSize;
NarListingRegularFile{ if (st.isExecutable)
.fileSize = st.fileSize, obj["executable"] = true;
.narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt, if (st.narOffset && *st.narOffset)
}, obj["narOffset"] = *st.narOffset;
}; break;
case SourceAccessor::Type::tDirectory: { case SourceAccessor::Type::tDirectory:
typename ListNarResult<deep>::Directory dir; obj["type"] = "directory";
for (const auto & [name, type] : accessor.readDirectory(path)) { {
if constexpr (deep) { obj["entries"] = json::object();
dir.entries.emplace(name, listNarImpl<true>(accessor, path / name)); json & res2 = obj["entries"];
} else { for (const auto & [name, type] : accessor->readDirectory(path)) {
dir.entries.emplace(name, fso::Opaque{}); if (recurse) {
res2[name] = listNar(accessor, path / name, true);
} else
res2[name] = json::object();
} }
} }
return dir; break;
}
case SourceAccessor::Type::tSymlink: case SourceAccessor::Type::tSymlink:
return typename ListNarResult<deep>::Symlink{ obj["type"] = "symlink";
.target = accessor.readLink(path), obj["target"] = accessor->readLink(path);
}; break;
case SourceAccessor::Type::tBlock: case SourceAccessor::Type::tBlock:
case SourceAccessor::Type::tChar: case SourceAccessor::Type::tChar:
case SourceAccessor::Type::tSocket: case SourceAccessor::Type::tSocket:
@@ -312,16 +314,7 @@ static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPat
case SourceAccessor::Type::tUnknown: case SourceAccessor::Type::tUnknown:
assert(false); // cannot happen for NARs assert(false); // cannot happen for NARs
} }
} return obj;
NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path)
{
return listNarImpl<true>(accessor, path);
}
ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path)
{
return listNarImpl<false>(accessor, path);
} }
} // namespace nix } // namespace nix
+1 -1
View File
@@ -172,7 +172,7 @@ void LocalStore::optimisePath_(
auto stLink = lstat(linkPath.string()); auto stLink = lstat(linkPath.string());
if (st.st_size != stLink.st_size || (repair && hash != ({ if (st.st_size != stLink.st_size || (repair && hash != ({
hashPath( hashPath(
makeFSSourceAccessor(linkPath), PosixSourceAccessor::createAtRoot(linkPath),
FileSerialisationMethod::NixArchive, FileSerialisationMethod::NixArchive,
HashAlgorithm::SHA256) HashAlgorithm::SHA256)
.hash; .hash;
+3 -3
View File
@@ -100,7 +100,7 @@ static nlohmann::json pathInfoToJSON(Store & store, const StorePathSet & storePa
nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs( nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
Store & store, Store & store,
const DerivationOptions<StorePath> & drvOptions, const DerivationOptions & drvOptions,
const StorePathSet & inputPaths, const StorePathSet & inputPaths,
const DerivationOutputs & outputs) const const DerivationOutputs & outputs) const
{ {
@@ -114,8 +114,8 @@ nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
json["outputs"] = std::move(outputsJson); json["outputs"] = std::move(outputsJson);
/* Handle exportReferencesGraph. */ /* Handle exportReferencesGraph. */
for (auto & [key, storePaths] : drvOptions.exportReferencesGraph) { for (auto & [key, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
json[key] = pathInfoToJSON(store, store.exportReferences(storePaths, inputPaths)); json[key] = pathInfoToJSON(store, store.exportReferences(storePaths, storePaths));
} }
return json; return json;
Loaded 100 of 166 files, more files were not shown because too many files have changed in this diff. Show more