Compare commits

..
Author SHA1 Message Date
Sergei Zimmerman b9ecd329ae treewide: Replace a lot of std::function with recursive lambdas via 'deducing this'
This is much simpler and has less overhead than a std::function, which does
type erasure.
2025-10-17 02:58:10 +03:00
98 changed files with 615 additions and 1853 deletions

No files matched your search

-1
View File
@@ -116,7 +116,6 @@ rec {
) nixComponentsInstrumented)
// lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
"${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests;
"${componentTestsPrefix}nix-json-schema-checks" = nixComponentsInstrumented.nix-json-schema-checks;
};
codeCoverage =
+1 -2
View File
@@ -88,7 +88,7 @@ manual = custom_target(
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
@4@ -r -L --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
@4@ -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
rm -rf @2@/manual
mv @2@/html @2@/manual
@@ -115,7 +115,6 @@ manual = custom_target(
builtins_md,
rl_next_generated,
summary_rl_next,
json_schema_generated_files,
nix_input,
],
output : [
-5
View File
@@ -12,7 +12,6 @@
rsync,
nix-cli,
changelog-d,
json-schema-for-humans,
officialRelease,
# Configuration Options
@@ -33,9 +32,6 @@ mkMesonDerivation (finalAttrs: {
fileset.difference
(fileset.unions [
../../.version
# For example JSON
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash
# Too many different types of files to filter for now
../../doc/manual
./.
@@ -59,7 +55,6 @@ mkMesonDerivation (finalAttrs: {
jq
python3
rsync
json-schema-for-humans
changelog-d
]
++ lib.optionals (!officialRelease) [
-2
View File
@@ -117,8 +117,6 @@
- [Architecture and Design](architecture/architecture.md)
- [Formats and Protocols](protocols/index.md)
- [JSON Formats](protocols/json/index.md)
- [File System Object](protocols/json/file-system-object.md)
- [Hash](protocols/json/hash.md)
- [Store Object Info](protocols/json/store-object-info.md)
- [Derivation](protocols/json/derivation.md)
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
-3
View File
@@ -1,6 +1,3 @@
# Process JSON schema documentation
subdir('protocols')
summary_rl_next = custom_target(
command : [
bash,
+118 -5
View File
@@ -1,7 +1,120 @@
{{#include derivation-v3-fixed.md}}
# Derivation JSON Format
<!--
## Raw Schema
> **Warning**
>
> This JSON format is currently
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
> and subject to change.
[JSON Schema for Derivation v3](schema/derivation-v3.json)
-->
The JSON serialization of a
[derivations](@docroot@/glossary.md#gloss-store-derivation)
is a JSON object with the following fields:
* `name`:
The name of the derivation.
This is used when calculating the store paths of the derivation's outputs.
* `version`:
Must be `3`.
This is a guard that allows us to continue evolving this format.
The choice of `3` is fairly arbitrary, but corresponds to this informal version:
- Version 0: A-Term format
- Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
- Version 2: Separate `method` and `hashAlgo` fields in output specs
- Version 3: Drop store dir from store paths, just include base name.
Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
* `outputs`:
Information about the output paths of the derivation.
This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
* `path`:
The output path, if it is known in advanced.
Otherwise, `null`.
* `method`:
For an output which will be [content addressed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
Valid method strings are:
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
Otherwise, `null`.
* `hashAlgo`:
For an output which will be [content addressed], the name of the hash algorithm used.
Valid algorithm strings are:
- `blake3`
- `md5`
- `sha1`
- `sha256`
- `sha512`
* `hash`:
For fixed-output derivations, the expected content hash in base-16.
> **Example**
>
> ```json
> "outputs": {
> "out": {
> "method": "nar",
> "hashAlgo": "sha256",
> "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
> }
> }
> ```
* `inputSrcs`:
A list of store paths on which this derivation depends.
> **Example**
>
> ```json
> "inputSrcs": [
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
> ]
> ```
* `inputDrvs`:
A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
> **Example**
>
> ```json
> "inputDrvs": {
> "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
> "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
> }
> ```
specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
* `system`:
The system type on which this derivation is to be built
(e.g. `x86_64-linux`).
* `builder`:
The absolute path of the program to be executed to run the build.
Typically this is the `bash` shell
(e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
* `args`:
The command-line arguments passed to the `builder`.
* `env`:
The environment passed to the `builder`.
* `structuredAttrs`:
[Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
Structured attributes are JSON, and thus embedded as-is.
@@ -1,21 +0,0 @@
{{#include file-system-object-v1-fixed.md}}
## Examples
### Simple
```json
{{#include schema/file-system-object-v1/simple.json}}
```
### Complex
```json
{{#include schema/file-system-object-v1/complex.json}}
```
<!--
## Raw Schema
[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
-->
@@ -1,15 +0,0 @@
# For some reason, backticks in the JSON schema are being escaped rather
# than being kept as intentional code spans. This removes all backtick
# escaping, which is an ugly solution, but one that is fine, because we
# are not using backticks for any other purpose.
s/\\`/`/g
# The way that semi-external references are rendered (i.e. ones to
# sibling schema files, as opposed to separate website ones, is not nice
# for humans. Replace it with a nice relative link within the manual
# instead.
#
# As we have more such relative links, more replacements of this nature
# should appear below.
s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
-33
View File
@@ -1,33 +0,0 @@
{{#include hash-v1-fixed.md}}
## Examples
### SHA-256 with Base64 encoding
```json
{{#include schema/hash-v1/sha256-base64.json}}
```
### SHA-256 with Base16 (hexadecimal) encoding
```json
{{#include schema/hash-v1/sha256-base16.json}}
```
### SHA-256 with Nix32 encoding
```json
{{#include schema/hash-v1/sha256-nix32.json}}
```
### BLAKE3 with Base64 encoding
```json
{{#include schema/hash-v1/blake3-base64.json}}
```
<!--
## Raw Schema
[JSON Schema for Hash v1](schema/hash-v1.json)
-->
@@ -1,17 +0,0 @@
# Configuration file for json-schema-for-humans
#
# https://github.com/coveooss/json-schema-for-humans/blob/main/docs/examples/examples_md_default/Configuration.md
template_name: md
show_toc: true
# impure timestamp and distracting
with_footer: false
recursive_detection_depth: 3
show_breadcrumbs: false
description_is_markdown: true
template_md_options:
properties_table_columns:
- Property
- Type
- Pattern
- Title/Description
@@ -1,75 +0,0 @@
# Tests in: ../../../../src/json-schema-checks
fs = import('fs')
# Find json-schema-for-humans if available
json_schema_for_humans = find_program('generate-schema-doc', required : false)
# Configuration for json-schema-for-humans
json_schema_config = files('json-schema-for-humans-config.yaml')
schemas = [
'file-system-object-v1',
'hash-v1',
'derivation-v3',
]
schema_files = files()
foreach schema_name : schemas
schema_files += files('schema' / schema_name + '.yaml')
endforeach
schema_outputs = []
foreach schema_name : schemas
schema_outputs += schema_name + '.md'
endforeach
json_schema_generated_files = []
# Generate markdown documentation from JSON schema
# Note: output must be just a filename, not a path
gen_file = custom_target(
schema_name + '-schema-docs.tmp',
command : [
json_schema_for_humans,
'--config-file',
json_schema_config,
meson.current_source_dir() / 'schema',
meson.current_build_dir(),
],
input : schema_files + [
json_schema_config,
],
output : schema_outputs,
capture : false,
build_by_default : true,
)
idx = 0
if json_schema_for_humans.found()
foreach schema_name : schemas
#schema_file = 'schema' / schema_name + '.yaml'
# There is one so-so hack, and one horrible hack being done here.
sedded_file = custom_target(
schema_name + '-schema-docs',
command : [
'sed',
'-f',
# Out of line to avoid https://github.com/mesonbuild/meson/issues/1564
files('fixup-json-schema-generated-doc.sed'),
'@INPUT@',
],
capture : true,
input : gen_file[idx],
output : schema_name + '-fixed.md',
)
idx += 1
json_schema_generated_files += [ sedded_file ]
endforeach
else
warning(
'json-schema-for-humans not found, skipping JSON schema documentation generation',
)
endif
@@ -1,178 +0,0 @@
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json
title: Derivation
description: |
Experimental JSON representation of a Nix derivation (version 3).
This schema describes the JSON representation of Nix's `Derivation` type.
> **Warning**
>
> This JSON format is currently
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
> and subject to change.
type: object
required:
- name
- version
- outputs
- inputSrcs
- inputDrvs
- system
- builder
- args
- env
properties:
name:
type: string
title: Derivation name
description: |
The name of the derivation.
Used when calculating store paths for the derivation’s outputs.
version:
const: 3
title: Format version (must be 3)
description: |
Must be `3`.
This is a guard that allows us to continue evolving this format.
The choice of `3` is fairly arbitrary, but corresponds to this informal version:
- Version 0: A-Term format
- Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
- Version 2: Separate `method` and `hashAlgo` fields in output specs
- Version 3: Drop store dir from store paths, just include base name.
Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
outputs:
type: object
title: Output specifications
description: |
Information about the output paths of the derivation.
This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described.
> **Example**
>
> ```json
> "outputs": {
> "out": {
> "method": "nar",
> "hashAlgo": "sha256",
> "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
> }
> }
> ```
additionalProperties:
"$ref": "#/$defs/output"
inputSrcs:
type: array
title: Input source paths
description: |
List of store paths on which this derivation depends.
> **Example**
>
> ```json
> "inputSrcs": [
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
> ]
> ```
items:
type: string
inputDrvs:
type: object
title: Input derivations
description: |
Mapping of derivation paths to lists of output names they provide.
> **Example**
>
> ```json
> "inputDrvs": {
> "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
> "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
> }
> ```
>
> specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
system:
type: string
title: Build system type
description: |
The system type on which this derivation is to be built
(e.g. `x86_64-linux`).
builder:
type: string
title: Build program path
description: |
Absolute path of the program used to perform the build.
Typically this is the `bash` shell
(e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
args:
type: array
title: Builder arguments
description: |
Command-line arguments passed to the `builder`.
items:
type: string
env:
type: object
title: Environment variables
description: |
Environment variables passed to the `builder`.
additionalProperties:
type: string
structuredAttrs:
title: Structured attributes
description: |
[Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
Structured attributes are JSON, and thus embedded as-is.
type: object
additionalProperties: true
"$defs":
output:
type: object
properties:
path:
type: string
title: Output path
description: |
The output path, if known in advance.
method:
type: string
title: Content addressing method
enum: [flat, nar, text, git]
description: |
For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
Valid method strings are:
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
hashAlgo:
title: Hash algorithm
"$ref": "./hash-v1.yaml#/$defs/algorithm"
hash:
type: string
title: Expected hash value
description: |
For fixed-output derivations, the expected content hash in base-16.
@@ -1 +0,0 @@
../../../../../../src/libutil-tests/data/memory-source-accessor
@@ -1,65 +0,0 @@
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
title: File System Object
description: |
This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
The schema is recursive because file system objects contain other file system objects.
type: object
required: ["type"]
properties:
type:
type: string
enum: ["regular", "symlink", "directory"]
# Enforce conditional structure based on `type`
anyOf:
- $ref: "#/$defs/regular"
required: ["type", "contents"]
- $ref: "#/$defs/symlink"
required: ["type", "target"]
- $ref: "#/$defs/directory"
required: ["type", "contents"]
"$defs":
regular:
title: Regular File
required: ["contents"]
properties:
type:
const: "regular"
contents:
type: string
description: Base64-encoded file contents
executable:
type: boolean
description: Whether the file is executable.
default: false
additionalProperties: false
symlink:
title: Symbolic Link
required: ["target"]
properties:
type:
const: "symlink"
target:
type: string
description: Target path of the symlink.
additionalProperties: false
directory:
title: Directory
required: ["contents"]
properties:
type:
const: "directory"
contents:
type: object
description: |
Map of names to nested file system objects (for type=directory)
additionalProperties:
$ref: "#"
additionalProperties: false
@@ -1 +0,0 @@
../../../../../../src/libutil-tests/data/hash/
@@ -1,54 +0,0 @@
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json
title: Hash
description: |
A cryptographic hash value used throughout Nix for content addressing and integrity verification.
This schema describes the JSON representation of Nix's `Hash` type.
type: object
properties:
algorithm:
"$ref": "#/$defs/algorithm"
format:
type: string
enum:
- base64
- nix32
- base16
- sri
title: Hash format
description: |
The encoding format of the hash value.
- `base64` uses standard Base64 encoding [RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4)
- `nix32` is Nix-specific base-32 encoding
- `base16` is lowercase hexadecimal
- `sri` is the [Subresource Integrity format](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity).
hash:
type: string
title: Hash
description: |
The encoded hash value, itself.
It is specified in the format specified by the `format` field.
It must be the right length for the hash algorithm specified in the `algorithm` field, also.
The hash value does not include any algorithm prefix.
required:
- algorithm
- format
- hash
additionalProperties: false
"$defs":
algorithm:
type: string
enum:
- blake3
- md5
- sha1
- sha256
- sha512
title: Hash algorithm
description: |
The hash algorithm used to compute the hash value.
`blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature.
-2
View File
@@ -1,2 +0,0 @@
# Process JSON schema documentation
subdir('json')
-4
View File
@@ -413,10 +413,6 @@
supportsCross = false;
};
"nix-json-schema-checks" = {
supportsCross = false;
};
"nix-perl-bindings" = {
supportsCross = false;
};
-1
View File
@@ -60,4 +60,3 @@ if get_option('unit-tests')
subproject('libflake-tests')
endif
subproject('nix-functional-tests')
subproject('json-schema-checks')
-5
View File
@@ -438,11 +438,6 @@ in
*/
nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; };
/**
JSON schema validation checks
*/
nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { };
nix-perl-bindings = callPackage ../src/perl/package.nix { };
/**
-1
View File
@@ -108,7 +108,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs
++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs
++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs
++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs
++ lib.optional (
!buildCanExecuteHost
# Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
-1
View File
@@ -62,7 +62,6 @@ let
"nix-cmd"
"nix-cli"
"nix-functional-tests"
"nix-json-schema-checks"
]
++ lib.optionals enableBindings [
"nix-perl-bindings"
-1
View File
@@ -1 +0,0 @@
../../.version
-1
View File
@@ -1 +0,0 @@
../../src/libstore-tests/data/derivation
@@ -1 +0,0 @@
../../src/libutil-tests/data/memory-source-accessor
-1
View File
@@ -1 +0,0 @@
../../src/libutil-tests/data/hash
-92
View File
@@ -1,92 +0,0 @@
# Run with:
# meson test --suite json-schema
# Run with: (without shell / configure)
# nix build .#nix-json-schema-checks
project(
'nix-json-schema-checks',
version : files('.version'),
meson_version : '>= 1.1',
license : 'LGPL-2.1-or-later',
)
fs = import('fs')
# Note: The 'jsonschema' package provides the 'jv' command
jv = find_program('jv', required : true)
# The schema directory is a committed symlink to the actual schema location
schema_dir = meson.current_source_dir() / 'schema'
# Get all example files
schemas = [
{
'stem' : 'file-system-object',
'schema' : schema_dir / 'file-system-object-v1.yaml',
'files' : [
'simple.json',
'complex.json',
],
},
{
'stem' : 'hash',
'schema' : schema_dir / 'hash-v1.yaml',
'files' : [
'sha256-base64.json',
'sha256-base16.json',
'sha256-nix32.json',
'blake3-base64.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml',
'files' : [
'dyn-dep-derivation.json',
'simple-derivation.json',
],
},
# # Not sure how to make subschema work
# {
# 'stem': 'derivation',
# 'schema': schema_dir / 'derivation-v3.yaml#output',
# 'files' : [
# 'output-caFixedFlat.json',
# 'output-caFixedNAR.json',
# 'output-caFixedText.json',
# 'output-caFloating.json',
# 'output-deferred.json',
# 'output-impure.json',
# 'output-inputAddressed.json',
# ],
# },
]
# Validate each example against the schema
foreach schema : schemas
stem = schema['stem']
schema_file = schema['schema']
if '#' not in schema_file
# Validate the schema itself against JSON Schema Draft 04
test(
stem + '-schema-valid',
jv,
args : [
'http://json-schema.org/draft-04/schema',
schema_file,
],
suite : 'json-schema',
)
endif
foreach example : schema['files']
test(
stem + '-example-' + fs.stem(example),
jv,
args : [
schema_file,
files(stem / example),
],
suite : 'json-schema',
)
endforeach
endforeach
-52
View File
@@ -1,52 +0,0 @@
# Run with: nix build .#nix-json-schema-checks
{
lib,
mkMesonDerivation,
meson,
ninja,
jsonschema,
# Configuration Options
version,
}:
mkMesonDerivation (finalAttrs: {
pname = "nix-json-schema-checks";
inherit version;
workDir = ./.;
fileset = lib.fileset.unions [
../../.version
../../doc/manual/source/protocols/json/schema
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash
../../src/libstore-tests/data/derivation
./.
];
outputs = [ "out" ];
passthru.externalNativeBuildInputs = [
jsonschema
];
nativeBuildInputs = [
meson
ninja
]
++ finalAttrs.passthru.externalNativeBuildInputs;
doCheck = true;
mesonCheckFlags = [ "--print-errorlogs" ];
postInstall = ''
touch $out
'';
meta = {
platforms = lib.platforms.all;
};
})
-1
View File
@@ -1 +0,0 @@
../../doc/manual/source/protocols/json/schema
+1 -3
View File
@@ -2160,9 +2160,7 @@ void EvalState::forceValueDeep(Value & v)
{
std::set<const Value *> seen;
std::function<void(Value & v)> recurse;
recurse = [&](Value & v) {
auto recurse = [&](this auto & recurse, Value & v) {
if (!seen.insert(&v).second)
return;
+2 -2
View File
@@ -199,8 +199,8 @@ static void fetchTree(
if (state.settings.pureEval && !input.isLocked()) {
if (input.getNarHash())
warn(
"Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
"This is not reproducible and will break after garbage collection or when shared.",
"Input '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. "
"This is deprecated since such inputs are verifiable but may not be reproducible.",
input.to_string());
else
state
+3 -4
View File
@@ -92,7 +92,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
std::istringstream tomlStream(std::string{toml});
auto visit = [&](auto & self, Value & v, toml::value t) -> void {
auto visit = [&](this auto & visit, Value & v, toml::value t) -> void {
switch (t.type()) {
case toml::value_t::table: {
auto table = toml::get<toml::table>(t);
@@ -100,7 +100,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
for (auto & elem : table) {
forceNoNullByte(elem.first);
self(self, attrs.alloc(elem.first), elem.second);
visit(attrs.alloc(elem.first), elem.second);
}
v.mkAttrs(attrs);
@@ -110,7 +110,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
auto list = state.buildList(array.size());
for (const auto & [n, v] : enumerate(list))
self(self, *(v = state.allocValue()), array[n]);
visit(*(v = state.allocValue()), array[n]);
v.mkList(list);
} break;
case toml::value_t::boolean:
@@ -155,7 +155,6 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
try {
visit(
visit,
val,
toml::parse(
tomlStream,
+21 -31
View File
@@ -469,41 +469,31 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
std::vector<FlakeRef> parents;
std::function<void(
const FlakeInputs & flakeInputs,
ref<Node> node,
const InputAttrPath & inputAttrPathPrefix,
std::shared_ptr<const Node> oldNode,
const InputAttrPath & followsPrefix,
const SourcePath & sourcePath,
bool trustLock)>
computeLocks;
computeLocks = [&](
/* The inputs of this node, either from flake.nix or
flake.lock. */
const FlakeInputs & flakeInputs,
/* The node whose locks are to be updated.*/
ref<Node> node,
/* The path to this node in the lock file graph. */
const InputAttrPath & inputAttrPathPrefix,
/* The old node, if any, from which locks can be
copied. */
std::shared_ptr<const Node> oldNode,
/* The prefix relative to which 'follows' should be
interpreted. When a node is initially locked, it's
relative to the node's flake; when it's already locked,
it's relative to the root of the lock file. */
const InputAttrPath & followsPrefix,
/* The source path of this node's flake. */
const SourcePath & sourcePath,
bool trustLock) {
auto computeLocks = [&](this auto & computeLocks,
/* The inputs of this node, either from flake.nix or
flake.lock. */
const FlakeInputs & flakeInputs,
/* The node whose locks are to be updated.*/
ref<Node> node,
/* The path to this node in the lock file graph. */
const InputAttrPath & inputAttrPathPrefix,
/* The old node, if any, from which locks can be
copied. */
std::shared_ptr<const Node> oldNode,
/* The prefix relative to which 'follows' should be
interpreted. When a node is initially locked, it's
relative to the node's flake; when it's already locked,
it's relative to the root of the lock file. */
const InputAttrPath & followsPrefix,
/* The source path of this node's flake. */
const SourcePath & sourcePath,
bool trustLock) -> void {
debug("computing lock file node '%s'", printInputAttrPath(inputAttrPathPrefix));
/* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */
std::function<void(const FlakeInput & input, const InputAttrPath & prefix)> addOverrides;
addOverrides = [&](const FlakeInput & input, const InputAttrPath & prefix) {
auto addOverrides =
[&](this auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void {
for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputAttrPath(prefix);
inputAttrPath.push_back(idOverride);
+5 -11
View File
@@ -77,8 +77,8 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann:
if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) {
if (lockedRef.input.getNarHash())
warn(
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
"This is not reproducible and will break after garbage collection or when shared.",
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. "
"This is deprecated since such inputs are verifiable but may not be reproducible.",
lockedRef.to_string());
else
throw Error(
@@ -149,9 +149,7 @@ LockFile::LockFile(const fetchers::Settings & fetchSettings, std::string_view co
std::map<std::string, ref<Node>> nodeMap;
std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
getInputs = [&](Node & node, const nlohmann::json & jsonNode) {
auto getInputs = [&](this auto & getInputs, Node & node, const nlohmann::json & jsonNode) -> void {
if (jsonNode.find("inputs") == jsonNode.end())
return;
for (auto & i : jsonNode["inputs"].items()) {
@@ -276,9 +274,7 @@ std::optional<FlakeRef> LockFile::isUnlocked(const fetchers::Settings & fetchSet
{
std::set<ref<const Node>> nodes;
std::function<void(ref<const Node> node)> visit;
visit = [&](ref<const Node> node) {
auto visit = [&](this auto & visit, ref<const Node> node) -> void {
if (!nodes.insert(node).second)
return;
for (auto & i : node->inputs)
@@ -332,9 +328,7 @@ std::map<InputAttrPath, Node::Edge> LockFile::getAllInputs() const
std::set<ref<Node>> done;
std::map<InputAttrPath, Node::Edge> res;
std::function<void(const InputAttrPath & prefix, ref<Node> node)> recurse;
recurse = [&](const InputAttrPath & prefix, ref<Node> node) {
auto recurse = [&](this auto & recurse, const InputAttrPath & prefix, ref<Node> node) -> void {
if (!done.insert(node).second)
return;
+39 -39
View File
@@ -14,7 +14,7 @@
namespace nix {
using namespace nlohmann;
using nlohmann::json;
class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest
{
@@ -51,44 +51,44 @@ using BothFixtures = ::testing::Types<DerivationAdvancedAttrsTest, CaDerivationA
TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures);
#define TEST_ATERM_JSON(STEM, NAME) \
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \
{ \
this->readTest(NAME ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
/* Use DRV file instead of C++ literal as source of truth. */ \
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
Derivation got = adl_serializer<Derivation>::from_json(encoded, this->mockXpSettings); \
EXPECT_EQ(got, expected); \
}); \
} \
\
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \
{ \
this->writeTest( \
NAME ".json", \
[&]() -> json { \
/* Use DRV file instead of C++ literal as source of truth. */ \
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
}, \
[](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
} \
\
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \
{ \
this->readTest(NAME ".drv", [&](auto encoded) { \
/* Use JSON file instead of C++ literal as source of truth. */ \
auto j = json::parse(readFile(this->goldenMaster(NAME ".json"))); \
auto expected = adl_serializer<Derivation>::from_json(j, this->mockXpSettings); \
auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \
EXPECT_EQ(static_cast<json>(got), static_cast<json>(expected)); \
EXPECT_EQ(got, expected); \
}); \
} \
\
#define TEST_ATERM_JSON(STEM, NAME) \
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \
{ \
this->readTest(NAME ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
/* Use DRV file instead of C++ literal as source of truth. */ \
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \
EXPECT_EQ(got, expected); \
}); \
} \
\
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \
{ \
this->writeTest( \
NAME ".json", \
[&]() -> json { \
/* Use DRV file instead of C++ literal as source of truth. */ \
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \
}, \
[](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
} \
\
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \
{ \
this->readTest(NAME ".drv", [&](auto encoded) { \
/* Use JSON file instead of C++ literal as source of truth. */ \
auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \
auto expected = Derivation::fromJSON(json, this->mockXpSettings); \
auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \
EXPECT_EQ(got.toJSON(), expected.toJSON()); \
EXPECT_EQ(got, expected); \
}); \
} \
\
/* No corresponding write test, because we need to read the drv to write the json file */
TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults");
+25 -14
View File
@@ -66,17 +66,23 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps)
FormatError);
}
#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \
TEST_P(FIXTURE, from_json) \
{ \
const auto & [name, expected] = GetParam(); \
readJsonTest(Path{"output-"} + name, expected, mockXpSettings); \
} \
\
TEST_P(FIXTURE, to_json) \
{ \
const auto & [name, value] = GetParam(); \
writeJsonTest("output-" + name, value); \
#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \
TEST_P(FIXTURE, from_json) \
{ \
const auto & [name, expected] = GetParam(); \
/* Don't use readJsonTest because we want to check experimental \
features. */ \
readTest(Path{"output-"} + name + ".json", [&](const auto & encoded_) { \
json j = json::parse(encoded_); \
DerivationOutput got = DerivationOutput::fromJSON(j, mockXpSettings); \
ASSERT_EQ(got, expected); \
}); \
} \
\
TEST_P(FIXTURE, to_json) \
{ \
const auto & [name, value] = GetParam(); \
writeJsonTest("output-" + name, value); \
}
struct DerivationOutputJsonTest : DerivationTest,
@@ -187,7 +193,13 @@ INSTANTIATE_TEST_SUITE_P(
TEST_P(FIXTURE, from_json) \
{ \
const auto & drv = GetParam(); \
readJsonTest(drv.name, drv, mockXpSettings); \
/* Don't use readJsonTest because we want to check experimental \
features. */ \
readTest(drv.name + ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \
ASSERT_EQ(got, drv); \
}); \
} \
\
TEST_P(FIXTURE, to_json) \
@@ -201,8 +213,7 @@ INSTANTIATE_TEST_SUITE_P(
const auto & drv = GetParam(); \
readTest(drv.name + ".drv", [&](auto encoded) { \
auto got = parseDerivation(*store, std::move(encoded), drv.name, mockXpSettings); \
using nlohmann::json; \
ASSERT_EQ(static_cast<json>(got), static_cast<json>(drv)); \
ASSERT_EQ(got.toJSON(), drv.toJSON()); \
ASSERT_EQ(got, drv); \
}); \
} \
+24 -50
View File
@@ -3,13 +3,13 @@
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "nix/util/tests/json-characterization.hh"
#include "nix/util/tests/characterization.hh"
#include "nix/store/tests/derived-path.hh"
#include "nix/store/tests/libstore.hh"
namespace nix {
class DerivedPathTest : public virtual CharacterizationTest, public LibStoreTest
class DerivedPathTest : public CharacterizationTest, public LibStoreTest
{
std::filesystem::path unitTestData = getUnitTestData() / "derived-path";
@@ -123,51 +123,25 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o))
using nlohmann::json;
struct SingleDerivedPathJsonTest : DerivedPathTest,
JsonCharacterizationTest<SingleDerivedPath>,
::testing::WithParamInterface<SingleDerivedPath>
{};
struct DerivedPathJsonTest : DerivedPathTest,
JsonCharacterizationTest<DerivedPath>,
::testing::WithParamInterface<DerivedPath>
{};
#define TEST_JSON(TYPE, NAME, VAL) \
static const TYPE NAME = VAL; \
\
TEST_F(TYPE##JsonTest, NAME##_from_json) \
{ \
readJsonTest(#NAME, NAME); \
} \
\
TEST_F(TYPE##JsonTest, NAME##_to_json) \
{ \
writeJsonTest(#NAME, NAME); \
}
#define TEST_JSON_XP_DYN(TYPE, NAME, VAL) \
static const TYPE NAME = VAL; \
\
TEST_F(TYPE##JsonTest, NAME##_from_json_throws_without_xp) \
{ \
std::optional<json> ret; \
readTest(#NAME ".json", [&](const auto & encoded_) { ret = json::parse(encoded_); }); \
if (ret) { \
EXPECT_THROW(nlohmann::adl_serializer<TYPE>::from_json(*ret), MissingExperimentalFeature); \
} \
} \
\
TEST_F(TYPE##JsonTest, NAME##_from_json) \
{ \
ExperimentalFeatureSettings xpSettings; \
xpSettings.set("experimental-features", "dynamic-derivations"); \
readJsonTest(#NAME, NAME, xpSettings); \
} \
\
TEST_F(TYPE##JsonTest, NAME##_to_json) \
{ \
writeJsonTest(#NAME, NAME); \
#define TEST_JSON(TYPE, NAME, VAL) \
static const TYPE NAME = VAL; \
\
TEST_F(DerivedPathTest, NAME##_from_json) \
{ \
readTest(#NAME ".json", [&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
TYPE got = static_cast<TYPE>(encoded); \
ASSERT_EQ(got, NAME); \
}); \
} \
\
TEST_F(DerivedPathTest, NAME##_to_json) \
{ \
writeTest( \
#NAME ".json", \
[&]() -> json { return static_cast<json>(NAME); }, \
[](const auto & file) { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
}
TEST_JSON(
@@ -182,7 +156,7 @@ TEST_JSON(
.output = "bar",
}));
TEST_JSON_XP_DYN(
TEST_JSON(
SingleDerivedPath,
single_built_built,
(SingleDerivedPath::Built{
@@ -205,7 +179,7 @@ TEST_JSON(
.outputs = OutputsSpec::Names{"bar", "baz"},
}));
TEST_JSON_XP_DYN(
TEST_JSON(
DerivedPath,
multi_built_built,
(DerivedPath::Built{
@@ -217,7 +191,7 @@ TEST_JSON_XP_DYN(
.outputs = OutputsSpec::Names{"baz", "quux"},
}));
TEST_JSON_XP_DYN(
TEST_JSON(
DerivedPath,
multi_built_built_wildcard,
(DerivedPath::Built{
+46 -36
View File
@@ -96,57 +96,67 @@ public:
}
}
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createProviderForProfile(const std::string & profile);
private:
Aws::Crt::ApiHandle apiHandle;
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
credentialProviderCache;
};
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
{
debug(
"[pid=%d] creating new AWS credential provider for profile '%s'",
getpid(),
profile.empty() ? "(default)" : profile.c_str());
if (profile.empty()) {
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
}
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
// This is safe because the underlying C library will copy this string
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
}
AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
{
// Get or create credential provider with caching
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider;
// Use try_emplace_and_cvisit for atomic get-or-create
// This prevents race conditions where multiple threads create providers
credentialProviderCache.try_emplace_and_cvisit(
profile,
nullptr,
[&](auto & kv) { provider = kv.second = createProviderForProfile(profile); },
[&](const auto & kv) { provider = kv.second; });
nullptr, // Placeholder - will be replaced in f1 before any thread can see it
[&](auto & kv) {
// f1: Called atomically during insertion with non-const reference
// Other threads are blocked until we finish, so nullptr is never visible
debug(
"[pid=%d] creating new AWS credential provider for profile '%s'",
getpid(),
profile.empty() ? "(default)" : profile.c_str());
if (!provider) {
credentialProviderCache.erase_if(profile, [](const auto & kv) {
[[maybe_unused]] auto [_, provider] = kv;
return !provider;
try {
if (profile.empty()) {
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
} else {
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
// This is safe because the underlying C library will copy this string
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
}
if (!kv.second) {
throw AwsAuthError(
"Failed to create AWS credentials provider for %s",
profile.empty() ? "default profile" : fmt("profile '%s'", profile));
}
provider = kv.second;
} catch (Error & e) {
// Exception during creation - remove the entry to allow retry
credentialProviderCache.erase(profile);
e.addTrace({}, "for AWS profile: %s", profile.empty() ? "(default)" : profile);
throw;
} catch (...) {
// Non-Error exception - still need to clean up
credentialProviderCache.erase(profile);
throw;
}
},
[&](const auto & kv) {
// f2: Called if key already exists (const reference)
provider = kv.second;
});
throw AwsAuthError(
"Failed to create AWS credentials provider for %s",
profile.empty() ? "default profile" : fmt("profile '%s'", profile));
}
return getCredentialsFromProvider(provider);
}
@@ -112,21 +112,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
if (failed)
continue;
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
co_await await(std::move(waitees));
trace("output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
}
worker.store.registerDrvOutput({*outputInfo, id});
trace("finished");
co_return amDone(ecSuccess);
co_return realisationFetched(std::move(waitees), outputInfo, sub);
}
/* None left. Terminate this goal and let someone else deal
@@ -144,6 +130,26 @@ Goal::Co DrvOutputSubstitutionGoal::init()
co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
}
Goal::Co DrvOutputSubstitutionGoal::realisationFetched(
Goals waitees, std::shared_ptr<const UnkeyedRealisation> outputInfo, nix::ref<nix::Store> sub)
{
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
co_await await(std::move(waitees));
trace("output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
}
worker.store.registerDrvOutput({*outputInfo, id});
trace("finished");
co_return amDone(ecSuccess);
}
std::string DrvOutputSubstitutionGoal::key()
{
return "a$" + std::string(id.to_string());
+47 -24
View File
@@ -1261,15 +1261,9 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
} // namespace nix
namespace nlohmann {
using namespace nix;
void adl_serializer<DerivationOutput>::to_json(json & res, const DerivationOutput & o)
nlohmann::json DerivationOutput::toJSON() const
{
res = nlohmann::json::object();
nlohmann::json res = nlohmann::json::object();
std::visit(
overloaded{
[&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; },
@@ -1295,11 +1289,12 @@ void adl_serializer<DerivationOutput>::to_json(json & res, const DerivationOutpu
res["impure"] = true;
},
},
o.raw);
raw);
return res;
}
DerivationOutput
adl_serializer<DerivationOutput>::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings)
DerivationOutput::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings)
{
std::set<std::string_view> keys;
auto & json = getObject(_json);
@@ -1367,18 +1362,18 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
}
}
void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
nlohmann::json Derivation::toJSON() const
{
res = nlohmann::json::object();
nlohmann::json res = nlohmann::json::object();
res["name"] = d.name;
res["name"] = name;
res["version"] = 3;
{
nlohmann::json & outputsObj = res["outputs"];
outputsObj = nlohmann::json::object();
for (auto & [outputName, output] : d.outputs) {
for (auto & [outputName, output] : outputs) {
outputsObj[outputName] = output;
}
}
@@ -1386,7 +1381,7 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
{
auto & inputsList = res["inputSrcs"];
inputsList = nlohmann::json ::array();
for (auto & input : d.inputSrcs)
for (auto & input : inputSrcs)
inputsList.emplace_back(input);
}
@@ -1406,22 +1401,24 @@ void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
{
auto & inputDrvsObj = res["inputDrvs"];
inputDrvsObj = nlohmann::json::object();
for (auto & [inputDrv, inputNode] : d.inputDrvs.map) {
for (auto & [inputDrv, inputNode] : inputDrvs.map) {
inputDrvsObj[inputDrv.to_string()] = doInput(inputNode);
}
}
}
res["system"] = d.platform;
res["builder"] = d.builder;
res["args"] = d.args;
res["env"] = d.env;
res["system"] = platform;
res["builder"] = builder;
res["args"] = args;
res["env"] = env;
if (d.structuredAttrs)
res["structuredAttrs"] = d.structuredAttrs->structuredAttrs;
if (structuredAttrs)
res["structuredAttrs"] = structuredAttrs->structuredAttrs;
return res;
}
Derivation adl_serializer<Derivation>::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings)
Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings)
{
using nlohmann::detail::value_t;
@@ -1437,7 +1434,7 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
try {
auto outputs = getObject(valueAt(json, "outputs"));
for (auto & [outputName, output] : outputs) {
res.outputs.insert_or_assign(outputName, adl_serializer<DerivationOutput>::from_json(output, xpSettings));
res.outputs.insert_or_assign(outputName, DerivationOutput::fromJSON(output, xpSettings));
}
} catch (Error & e) {
e.addTrace({}, "while reading key 'outputs'");
@@ -1493,4 +1490,30 @@ Derivation adl_serializer<Derivation>::from_json(const json & _json, const Exper
return res;
}
} // namespace nix
namespace nlohmann {
using namespace nix;
DerivationOutput adl_serializer<DerivationOutput>::from_json(const json & json)
{
return DerivationOutput::fromJSON(json);
}
void adl_serializer<DerivationOutput>::to_json(json & json, const DerivationOutput & c)
{
json = c.toJSON();
}
Derivation adl_serializer<Derivation>::from_json(const json & json)
{
return Derivation::fromJSON(json);
}
void adl_serializer<Derivation>::to_json(json & json, const Derivation & c)
{
json = c.toJSON();
}
} // namespace nlohmann
+6 -8
View File
@@ -6,15 +6,14 @@ namespace nix {
template<typename V>
typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const SingleDerivedPath & k)
{
std::function<ChildNode &(const SingleDerivedPath &)> initIter;
initIter = [&](const auto & k) -> auto & {
auto initIter = [&](this auto & initIter, const auto & k) -> ChildNode & {
return std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & bo) -> auto & {
[&](const SingleDerivedPath::Opaque & bo) -> ChildNode & {
// will not overwrite if already there
return map[bo.path];
},
[&](const SingleDerivedPath::Built & bfd) -> auto & {
[&](const SingleDerivedPath::Built & bfd) -> ChildNode & {
auto & n = initIter(*bfd.drvPath);
return n.childMap[bfd.output];
},
@@ -27,15 +26,14 @@ typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const Sing
template<typename V>
typename DerivedPathMap<V>::ChildNode * DerivedPathMap<V>::findSlot(const SingleDerivedPath & k)
{
std::function<ChildNode *(const SingleDerivedPath &)> initIter;
initIter = [&](const auto & k) {
auto initIter = [&](this auto & initIter, const auto & k) -> ChildNode * {
return std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & bo) {
[&](const SingleDerivedPath::Opaque & bo) -> ChildNode * {
auto it = map.find(bo.path);
return it != map.end() ? &it->second : nullptr;
},
[&](const SingleDerivedPath::Built & bfd) {
[&](const SingleDerivedPath::Built & bfd) -> ChildNode * {
auto * n = initIter(*bfd.drvPath);
if (!n)
return (ChildNode *) nullptr;
+8 -15
View File
@@ -252,26 +252,20 @@ void adl_serializer<DerivedPath::Built>::to_json(json & json, const DerivedPath:
};
}
SingleDerivedPath::Built
adl_serializer<SingleDerivedPath::Built>::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings)
SingleDerivedPath::Built adl_serializer<SingleDerivedPath::Built>::from_json(const json & json0)
{
auto & json = getObject(json0);
auto drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath")));
drvRequireExperiment(*drvPath, xpSettings);
return {
.drvPath = std::move(drvPath),
.drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath"))),
.output = getString(valueAt(json, "output")),
};
}
DerivedPath::Built
adl_serializer<DerivedPath::Built>::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings)
DerivedPath::Built adl_serializer<DerivedPath::Built>::from_json(const json & json0)
{
auto & json = getObject(json0);
auto drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath")));
drvRequireExperiment(*drvPath, xpSettings);
return {
.drvPath = std::move(drvPath),
.drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath"))),
.outputs = adl_serializer<OutputsSpec>::from_json(valueAt(json, "outputs")),
};
}
@@ -286,21 +280,20 @@ void adl_serializer<DerivedPath>::to_json(json & json, const DerivedPath & sdp)
std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw());
}
SingleDerivedPath
adl_serializer<SingleDerivedPath>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
SingleDerivedPath adl_serializer<SingleDerivedPath>::from_json(const json & json)
{
if (json.is_string())
return static_cast<SingleDerivedPath::Opaque>(json);
else
return adl_serializer<SingleDerivedPath::Built>::from_json(json, xpSettings);
return static_cast<SingleDerivedPath::Built>(json);
}
DerivedPath adl_serializer<DerivedPath>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
DerivedPath adl_serializer<DerivedPath>::from_json(const json & json)
{
if (json.is_string())
return static_cast<DerivedPath::Opaque>(json);
else
return adl_serializer<DerivedPath::Built>::from_json(json, xpSettings);
return static_cast<DerivedPath::Built>(json);
}
} // namespace nlohmann
@@ -32,6 +32,8 @@ public:
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
Co init();
Co
realisationFetched(Goals waitees, std::shared_ptr<const UnkeyedRealisation> outputInfo, nix::ref<nix::Store> sub);
void timedOut(Error && ex) override
{
+13 -2
View File
@@ -134,6 +134,13 @@ struct DerivationOutput
*/
std::optional<StorePath>
path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
nlohmann::json toJSON() const;
/**
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivationOutput
fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
};
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@@ -383,6 +390,10 @@ struct Derivation : BasicDerivation
{
}
nlohmann::json toJSON() const;
static Derivation
fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
bool operator==(const Derivation &) const = default;
// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
// auto operator <=> (const Derivation &) const = default;
@@ -526,5 +537,5 @@ std::string hashPlaceholder(const OutputNameView outputName);
} // namespace nix
JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)
JSON_IMPL_WITH_XP_FEATURES(nix::Derivation)
JSON_IMPL(nix::DerivationOutput)
JSON_IMPL(nix::Derivation)
@@ -299,7 +299,7 @@ void drvRequireExperiment(
} // namespace nix
JSON_IMPL(nix::SingleDerivedPath::Opaque)
JSON_IMPL_WITH_XP_FEATURES(nix::SingleDerivedPath::Built)
JSON_IMPL_WITH_XP_FEATURES(nix::SingleDerivedPath)
JSON_IMPL_WITH_XP_FEATURES(nix::DerivedPath::Built)
JSON_IMPL_WITH_XP_FEATURES(nix::DerivedPath)
JSON_IMPL(nix::SingleDerivedPath::Built)
JSON_IMPL(nix::SingleDerivedPath)
JSON_IMPL(nix::DerivedPath::Built)
JSON_IMPL(nix::DerivedPath)
+17 -6
View File
@@ -113,7 +113,7 @@ boost = dependency(
# put in `deps_other`.
deps_other += boost
curl = dependency('libcurl', 'curl', version : '>= 7.75.0')
curl = dependency('libcurl', 'curl')
deps_private += curl
# seccomp only makes sense on Linux
@@ -142,16 +142,27 @@ deps_public += nlohmann_json
sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19')
deps_private += sqlite
s3_aws_auth = get_option('s3-aws-auth')
aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : s3_aws_auth)
# Curl-based S3 store support
# Check if curl supports AWS SigV4 (requires >= 7.75.0)
curl_supports_aws_sigv4 = curl.version().version_compare('>= 7.75.0')
# AWS CRT C++ for lightweight credential management
aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : false)
if s3_aws_auth.enabled()
curl_s3_store_opt = get_option('curl-s3-store').require(
curl_supports_aws_sigv4,
error_message : 'curl-based S3 support requires curl >= 7.75.0',
).require(
aws_crt_cpp.found(),
error_message : 'curl-based S3 support requires aws-crt-cpp',
)
if curl_s3_store_opt.enabled()
deps_other += aws_crt_cpp
aws_c_common = cxx.find_library('aws-c-common', required : true)
deps_other += aws_c_common
endif
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())
configdata_pub.set('NIX_WITH_AWS_AUTH', curl_s3_store_opt.enabled().to_int())
subdir('nix-meson-build-support/generate-header')
@@ -335,7 +346,7 @@ sources = files(
)
# AWS credentials code requires AWS CRT, so only compile when enabled
if s3_aws_auth.enabled()
if curl_s3_store_opt.enabled()
sources += files('aws-creds.cc')
endif
+3 -2
View File
@@ -35,7 +35,8 @@ option(
)
option(
's3-aws-auth',
'curl-s3-store',
type : 'feature',
description : 'build support for AWS authentication with S3',
value : 'disabled',
description : 'Enable curl-based S3 binary cache store support (requires aws-crt-cpp and curl >= 7.75.0)',
)
+1 -3
View File
@@ -146,9 +146,7 @@ struct NarAccessor : public SourceAccessor
{
using json = nlohmann::json;
std::function<void(NarMember &, const json &)> recurse;
recurse = [&](NarMember & member, const json & v) {
auto recurse = [&](this auto & recurse, NarMember & member, const json & v) -> void {
std::string type = v["type"];
if (type == "directory") {
+1 -1
View File
@@ -75,7 +75,7 @@ mkMesonLibrary (finalAttrs: {
mesonFlags = [
(lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
(lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
(lib.mesonEnable "s3-aws-auth" withAWS)
(lib.mesonEnable "curl-s3-store" withAWS)
]
++ lib.optionals stdenv.hostPlatform.isLinux [
(lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
@@ -24,12 +24,12 @@ struct JsonCharacterizationTest : virtual CharacterizationTest
* @param test hook that takes the contents of the file and does the
* actual work
*/
void readJsonTest(PathView testStem, const T & expected, auto... args)
void readJsonTest(PathView testStem, const T & expected)
{
using namespace nlohmann;
readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) {
auto encoded = json::parse(encodedRaw);
T decoded = adl_serializer<T>::from_json(encoded, args...);
T decoded = adl_serializer<T>::from_json(encoded);
ASSERT_EQ(decoded, expected);
});
}
-18
View File
@@ -1,18 +0,0 @@
#include "nix/util/alignment.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(alignUp, value)
{
for (uint64_t i = 1; i <= 8; ++i)
EXPECT_EQ(alignUp(i, 8), 8);
}
TEST(alignUp, notAPowerOf2)
{
ASSERT_DEATH({ alignUp(1u, 42); }, "alignment must be a power of 2");
}
} // namespace nix
@@ -1,5 +0,0 @@
{
"algorithm": "blake3",
"format": "base64",
"hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA="
}
@@ -1,5 +0,0 @@
{
"algorithm": "sha256",
"format": "base16",
"hash": "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"
}
@@ -1,5 +0,0 @@
{
"algorithm": "sha256",
"format": "base64",
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
}
@@ -1,5 +0,0 @@
{
"algorithm": "sha256",
"format": "nix32",
"hash": "0fz12qc1nillhvhw6bvs4ka18789x8dqaipjb316x4aqdkvw5r7h"
}
-5
View File
@@ -1,5 +0,0 @@
{
"algorithm": "sha256",
"format": "base64",
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
}
@@ -1,24 +0,0 @@
{
"contents": {
"bar": {
"contents": {
"baz": {
"contents": "Z29vZCBkYXksCg==",
"executable": true,
"type": "regular"
},
"quux": {
"target": "/over/there",
"type": "symlink"
}
},
"type": "directory"
},
"foo": {
"contents": "aGVsbG8K",
"executable": false,
"type": "regular"
}
},
"type": "directory"
}
@@ -1,5 +0,0 @@
{
"contents": "YXNkZg==",
"executable": false,
"type": "regular"
}
+35 -9
View File
@@ -224,15 +224,42 @@ TEST_F(GitTest, tree_sha256_write)
});
}
namespace memory_source_accessor {
extern ref<MemorySourceAccessor> exampleComplex();
}
TEST_F(GitTest, both_roundrip)
{
auto files = memory_source_accessor::exampleComplex();
using File = MemorySourceAccessor::File;
auto files = make_ref<MemorySourceAccessor>();
files->root = File::Directory{
.contents{
{
"foo",
File::Regular{
.contents = "hello\n\0\n\tworld!",
},
},
{
"bar",
File::Directory{
.contents =
{
{
"baz",
File::Regular{
.executable = true,
.contents = "good day,\n\0\n\tworld!",
},
},
{
"quux",
File::Symlink{
.target = "/over/there",
},
},
},
},
},
},
};
for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) {
std::map<Hash, std::string> cas;
@@ -257,8 +284,7 @@ TEST_F(GitTest, both_roundrip)
MemorySink sinkFiles2{*files2};
std::function<void(const CanonPath, const Hash &, BlobMode)> mkSinkHook;
mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) {
auto mkSinkHook = [&](this auto & mkSinkHook, auto prefix, auto & hash, auto blobMode) -> void {
StringSource in{cas[hash]};
parse(
sinkFiles2,
+9 -102
View File
@@ -4,30 +4,30 @@
#include <nlohmann/json.hpp>
#include "nix/util/hash.hh"
#include "nix/util/tests/json-characterization.hh"
#include "nix/util/tests/characterization.hh"
namespace nix {
class HashTest : public virtual CharacterizationTest
class HashTest : public CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "hash";
public:
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
struct BLAKE3HashTest : virtual HashTest
class BLAKE3HashTest : public HashTest
{
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
void SetUp() override
{
mockXpSettings.set("experimental-features", "blake3-hashes");
@@ -203,97 +203,4 @@ TEST(hashFormat, testParseHashFormatOptException)
{
ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt);
}
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
using nlohmann::json;
struct HashJsonTest : virtual HashTest,
JsonCharacterizationTest<Hash>,
::testing::WithParamInterface<std::pair<std::string_view, Hash>>
{};
struct HashJsonParseOnlyTest : virtual HashTest,
JsonCharacterizationTest<Hash>,
::testing::WithParamInterface<std::pair<std::string_view, Hash>>
{};
struct BLAKE3HashJsonTest : virtual HashTest,
BLAKE3HashTest,
JsonCharacterizationTest<Hash>,
::testing::WithParamInterface<std::pair<std::string_view, Hash>>
{};
TEST_P(HashJsonTest, from_json)
{
auto & [name, expected] = GetParam();
readJsonTest(name, expected);
}
TEST_P(HashJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
TEST_P(HashJsonParseOnlyTest, from_json)
{
auto & [name, expected] = GetParam();
readJsonTest(name, expected);
}
TEST_P(BLAKE3HashJsonTest, from_json)
{
auto & [name, expected] = GetParam();
readJsonTest(name, expected, mockXpSettings);
}
TEST_P(BLAKE3HashJsonTest, to_json)
{
auto & [name, expected] = GetParam();
writeJsonTest(name, expected);
}
// Round-trip tests (from_json + to_json) for base64 format only
// (to_json always outputs base64)
INSTANTIATE_TEST_SUITE_P(
HashJSON,
HashJsonTest,
::testing::Values(
std::pair{
"simple",
hashString(HashAlgorithm::SHA256, "asdf"),
},
std::pair{
"sha256-base64",
hashString(HashAlgorithm::SHA256, "asdf"),
}));
// Parse-only tests for non-base64 formats
// These verify C++ can deserialize other formats correctly
INSTANTIATE_TEST_SUITE_P(
HashJSONParseOnly,
HashJsonParseOnlyTest,
::testing::Values(
std::pair{
"sha256-base16",
hashString(HashAlgorithm::SHA256, "asdf"),
},
std::pair{
"sha256-nix32",
hashString(HashAlgorithm::SHA256, "asdf"),
}));
INSTANTIATE_TEST_SUITE_P(BLAKE3HashJSONParseOnly, BLAKE3HashJsonTest, ([] {
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "blake3-hashes");
return ::testing::Values(
std::pair{
"blake3-base64",
hashString(HashAlgorithm::BLAKE3, "asdf", mockXpSettings),
});
}()));
} // namespace nix
-116
View File
@@ -1,116 +0,0 @@
#include <string_view>
#include "nix/util/bytes.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
namespace memory_source_accessor {
using File = MemorySourceAccessor::File;
ref<MemorySourceAccessor> exampleSimple()
{
auto sc = make_ref<MemorySourceAccessor>();
sc->root = File{File::Regular{
.executable = false,
.contents = to_owned(as_bytes("asdf")),
}};
return sc;
}
ref<MemorySourceAccessor> exampleComplex()
{
auto files = make_ref<MemorySourceAccessor>();
files->root = File::Directory{
.contents{
{
"foo",
File::Regular{
.contents = to_owned(as_bytes("hello\n\0\n\tworld!")),
},
},
{
"bar",
File::Directory{
.contents =
{
{
"baz",
File::Regular{
.executable = true,
.contents = to_owned(as_bytes("good day,\n\0\n\tworld!")),
},
},
{
"quux",
File::Symlink{
.target = "/over/there",
},
},
},
},
},
},
};
return files;
}
} // namespace memory_source_accessor
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
class MemorySourceAccessorTest : public virtual CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor";
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
using nlohmann::json;
struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest,
JsonCharacterizationTest<MemorySourceAccessor>,
::testing::WithParamInterface<std::pair<std::string_view, MemorySourceAccessor>>
{};
TEST_P(MemorySourceAccessorJsonTest, from_json)
{
auto & [name, expected] = GetParam();
/* Cannot use `readJsonTest` because need to compare `root` field of
the source accessors for equality. */
readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
auto encoded = json::parse(encodedRaw);
auto decoded = static_cast<MemorySourceAccessor>(encoded);
ASSERT_EQ(decoded.root, expected.root);
});
}
TEST_P(MemorySourceAccessorJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
INSTANTIATE_TEST_SUITE_P(
MemorySourceAccessorJSON,
MemorySourceAccessorJsonTest,
::testing::Values(
std::pair{
"simple",
*memory_source_accessor::exampleSimple(),
},
std::pair{
"complex",
*memory_source_accessor::exampleComplex(),
}));
} // namespace nix
-2
View File
@@ -44,7 +44,6 @@ config_priv_h = configure_file(
subdir('nix-meson-build-support/common')
sources = files(
'alignment.cc',
'archive.cc',
'args.cc',
'base-n.cc',
@@ -63,7 +62,6 @@ sources = files(
'json-utils.cc',
'logging.cc',
'lru-cache.cc',
'memory-source-accessor.cc',
'monitorfdhup.cc',
'nix_api_util.cc',
'nix_api_util_internal.cc',
+1 -7
View File
@@ -6,7 +6,6 @@
#include <strings.h> // for strcasecmp
#include "nix/util/archive.hh"
#include "nix/util/alignment.hh"
#include "nix/util/config-global.hh"
#include "nix/util/posix-source-accessor.hh"
#include "nix/util/source-path.hh"
@@ -133,11 +132,6 @@ static void parseContents(CreateRegularFileSink & sink, Source & source)
sink.preallocateContents(size);
if (sink.skipContents) {
source.skip(alignUp(size, 8));
return;
}
uint64_t left = size;
std::array<char, 65536> buf;
@@ -172,7 +166,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath
auto expectTag = [&](std::string_view expected) {
auto tag = getString();
if (tag != expected)
throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024));
throw badArchive("expected tag '%s', got '%s'", expected, tag);
};
expectTag("(");
-2
View File
@@ -196,8 +196,6 @@ void NullFileSystemObjectSink::createRegularFile(
void isExecutable() override {}
} crf;
crf.skipContents = true;
// Even though `NullFileSystemObjectSink` doesn't do anything, it's important
// that we call the function, to e.g. advance the parser using this
// sink.
+8 -40
View File
@@ -13,7 +13,6 @@
#include "nix/util/split.hh"
#include "nix/util/base-n.hh"
#include "nix/util/base-nix-32.hh"
#include "nix/util/json-utils.hh"
#include <sys/types.h>
#include <sys/stat.h>
@@ -142,13 +141,9 @@ static HashFormat baseFromSize(std::string_view rest, HashAlgorithm algo)
*
* @param rest the string view to parse. Must not include any `<algo>(:|-)` prefix.
*/
static Hash parseLowLevel(
std::string_view rest,
HashAlgorithm algo,
DecodeNamePair pair,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
static Hash parseLowLevel(std::string_view rest, HashAlgorithm algo, DecodeNamePair pair)
{
Hash res{algo, xpSettings};
Hash res{algo};
std::string d;
try {
d = pair.decode(rest);
@@ -249,10 +244,9 @@ Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
return parseExplicitFormatUnprefixed(s, algo, baseFromSize(s, algo));
}
Hash Hash::parseExplicitFormatUnprefixed(
std::string_view s, HashAlgorithm algo, HashFormat format, const ExperimentalFeatureSettings & xpSettings)
Hash Hash::parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat format)
{
return parseLowLevel(s, algo, baseExplicit(format), xpSettings);
return parseLowLevel(s, algo, baseExplicit(format));
}
Hash Hash::random(HashAlgorithm algo)
@@ -452,12 +446,10 @@ std::string_view printHashFormat(HashFormat HashFormat)
}
}
std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s, const ExperimentalFeatureSettings & xpSettings)
std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
{
if (s == "blake3") {
xpSettings.require(Xp::BLAKE3Hashes);
if (s == "blake3")
return HashAlgorithm::BLAKE3;
}
if (s == "md5")
return HashAlgorithm::MD5;
if (s == "sha1")
@@ -469,9 +461,9 @@ std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s, const Experime
return std::nullopt;
}
HashAlgorithm parseHashAlgo(std::string_view s, const ExperimentalFeatureSettings & xpSettings)
HashAlgorithm parseHashAlgo(std::string_view s)
{
auto opt_h = parseHashAlgoOpt(s, xpSettings);
auto opt_h = parseHashAlgoOpt(s);
if (opt_h)
return *opt_h;
else
@@ -499,27 +491,3 @@ std::string_view printHashAlgo(HashAlgorithm ha)
}
} // namespace nix
namespace nlohmann {
using namespace nix;
Hash adl_serializer<Hash>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
{
auto & obj = getObject(json);
auto algo = parseHashAlgo(getString(valueAt(obj, "algorithm")), xpSettings);
auto format = parseHashFormat(getString(valueAt(obj, "format")));
auto & hashS = getString(valueAt(obj, "hash"));
return Hash::parseExplicitFormatUnprefixed(hashS, algo, format, xpSettings);
}
void adl_serializer<Hash>::to_json(json & json, const Hash & hash)
{
json = {
{"format", printHashFormat(HashFormat::Base64)},
{"algorithm", printHashAlgo(hash.algo)},
{"hash", hash.to_string(HashFormat::Base64, false)},
};
}
} // namespace nlohmann
-23
View File
@@ -1,23 +0,0 @@
#pragma once
///@file
#include <cassert>
#include <type_traits>
#include <cstdint>
#include <bit>
namespace nix {
/// Aligns val upwards to be a multiple of alignment.
///
/// @pre alignment must be a power of 2.
template<typename T>
requires std::is_unsigned_v<T>
constexpr T alignUp(T val, unsigned alignment)
{
assert(std::has_single_bit(alignment) && "alignment must be a power of 2");
T mask = ~(T{alignment} - 1u);
return (val + alignment - 1) & mask;
}
} // namespace nix
-41
View File
@@ -1,41 +0,0 @@
#pragma once
///@file
#include <string_view>
#include <span>
#include <vector>
namespace nix {
static inline std::span<const std::byte> as_bytes(std::string_view sv) noexcept
{
return std::span<const std::byte>{
reinterpret_cast<const std::byte *>(sv.data()),
sv.size(),
};
}
static inline std::vector<std::byte> to_owned(std::span<const std::byte> bytes)
{
return std::vector<std::byte>{
bytes.begin(),
bytes.end(),
};
}
/**
* @note this should be avoided, as arbitrary binary data in strings
* views, while allowed, is not really proper. Generally this should
* only be used as a stop-gap with other definitions that themselves
* should be converted to accept `std::span<const std::byte>` or
* similar, directly.
*/
static inline std::string_view to_str(std::span<const std::byte> sp)
{
return std::string_view{
reinterpret_cast<const char *>(sp.data()),
sp.size(),
};
}
} // namespace nix
-8
View File
@@ -14,14 +14,6 @@ namespace nix {
*/
struct CreateRegularFileSink : Sink
{
/**
* If set to true, the sink will not be called with the contents
* of the file. `preallocateContents()` will still be called to
* convey the file size. Useful for sinks that want to efficiently
* discard the contents of the file.
*/
bool skipContents = false;
virtual void isExecutable() = 0;
/**
+3 -16
View File
@@ -5,7 +5,6 @@
#include "nix/util/types.hh"
#include "nix/util/serialise.hh"
#include "nix/util/file-system.hh"
#include "nix/util/json-impls.hh"
namespace nix {
@@ -98,11 +97,7 @@ struct Hash
* @param explicitFormat cannot be SRI, but must be one of the
* "bases".
*/
static Hash parseExplicitFormatUnprefixed(
std::string_view s,
HashAlgorithm algo,
HashFormat explicitFormat,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
static Hash parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat explicitFormat);
static Hash parseSRI(std::string_view original);
@@ -193,14 +188,12 @@ std::string_view printHashFormat(HashFormat hashFormat);
/**
* Parse a string representing a hash algorithm.
*/
HashAlgorithm
parseHashAlgo(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
HashAlgorithm parseHashAlgo(std::string_view s);
/**
* Will return nothing on parse error
*/
std::optional<HashAlgorithm>
parseHashAlgoOpt(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s);
/**
* And the reverse.
@@ -228,10 +221,6 @@ public:
HashResult currentHash();
};
template<>
struct json_avoids_null<Hash> : std::true_type
{};
} // namespace nix
template<>
@@ -252,5 +241,3 @@ inline std::size_t hash_value(const Hash & hash)
}
} // namespace nix
JSON_IMPL_WITH_XP_FEATURES(Hash)
@@ -3,8 +3,6 @@
#include <nlohmann/json_fwd.hpp>
#include "nix/util/experimental-features.hh"
// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
#define JSON_IMPL(TYPE) \
namespace nlohmann { \
@@ -16,15 +14,3 @@
static void to_json(json & json, const TYPE & t); \
}; \
}
#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \
namespace nlohmann { \
using namespace nix; \
template<> \
struct adl_serializer<TYPE> \
{ \
static TYPE \
from_json(const json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); \
static void to_json(json & json, const TYPE & t); \
}; \
}
@@ -4,7 +4,6 @@
#include "nix/util/source-path.hh"
#include "nix/util/fs-sink.hh"
#include "nix/util/variant-wrapper.hh"
#include "nix/util/json-impls.hh"
namespace nix {
@@ -26,7 +25,7 @@ struct MemorySourceAccessor : virtual SourceAccessor
struct Regular
{
bool executable = false;
std::vector<std::byte> contents;
std::string contents;
bool operator==(const Regular &) const = default;
auto operator<=>(const Regular &) const = default;
@@ -87,13 +86,7 @@ struct MemorySourceAccessor : virtual SourceAccessor
*/
File * open(const CanonPath & path, std::optional<File> create);
SourcePath addFile(CanonPath path, std::vector<std::byte> && contents);
/**
* Small wrapper of the other `addFile`, purely for convenience when
* the file in question to be added is a string.
*/
SourcePath addFile(CanonPath path, std::string_view contents);
SourcePath addFile(CanonPath path, std::string && contents);
};
inline bool MemorySourceAccessor::File::Directory::operator==(
@@ -128,30 +121,4 @@ struct MemorySink : FileSystemObjectSink
void createSymlink(const CanonPath & path, const std::string & target) override;
};
template<>
struct json_avoids_null<MemorySourceAccessor::File::Regular> : std::true_type
{};
template<>
struct json_avoids_null<MemorySourceAccessor::File::Directory> : std::true_type
{};
template<>
struct json_avoids_null<MemorySourceAccessor::File::Symlink> : std::true_type
{};
template<>
struct json_avoids_null<MemorySourceAccessor::File> : std::true_type
{};
template<>
struct json_avoids_null<MemorySourceAccessor> : std::true_type
{};
} // namespace nix
JSON_IMPL(MemorySourceAccessor::File::Regular)
JSON_IMPL(MemorySourceAccessor::File::Directory)
JSON_IMPL(MemorySourceAccessor::File::Symlink)
JSON_IMPL(MemorySourceAccessor::File)
JSON_IMPL(MemorySourceAccessor)
-2
View File
@@ -4,7 +4,6 @@ include_dirs = [ include_directories('../..') ]
headers = files(
'abstract-setting-to-json.hh',
'alignment.hh',
'ansicolor.hh',
'archive.hh',
'args.hh',
@@ -12,7 +11,6 @@ headers = files(
'array-from-string-literal.hh',
'base-n.hh',
'base-nix-32.hh',
'bytes.hh',
'callback.hh',
'canon-path.hh',
'checked-arithmetic.hh',
@@ -97,8 +97,6 @@ struct Source
void drainInto(Sink & sink);
std::string drain();
virtual void skip(size_t len);
};
/**
@@ -179,7 +177,6 @@ struct FdSource : BufferedSource
Descriptor fd;
size_t read = 0;
BackedStringView endOfFileError{"unexpected end-of-file"};
bool isSeekable = true;
FdSource()
: fd(INVALID_DESCRIPTOR)
@@ -203,8 +200,6 @@ struct FdSource : BufferedSource
*/
bool hasData();
void skip(size_t len) override;
protected:
size_t readUnbuffered(char * data, size_t len) override;
private:
@@ -241,4 +241,10 @@ ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root);
*/
ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors);
/**
* Creates a new source accessor which is confined to the subdirectory
* of the given source accessor.
*/
ref<SourceAccessor> projectSubdirSourceAccessor(ref<SourceAccessor>, CanonPath subdirectory);
} // namespace nix
+1 -3
View File
@@ -14,9 +14,7 @@ std::vector<T> topoSort(
std::vector<T> sorted;
decltype(items) visited, parents;
std::function<void(const T & path, const T * parent)> dfsVisit;
dfsVisit = [&](const T & path, const T * parent) {
auto dfsVisit = [&](this auto & dfsVisit, const T & path, const T * parent) -> void {
if (parents.count(path))
throw makeCycleError(path, *parent);
-11
View File
@@ -408,17 +408,6 @@ struct VerbatimURL
[](const ParsedURL & url) -> std::string_view { return url.scheme; }},
raw);
}
/**
* Get the last non-empty path segment from the URL.
*
* This is useful for extracting filenames from URLs.
* For example, "https://example.com/path/to/file.txt?query=value"
* returns "file.txt".
*
* @return The last non-empty path segment, or std::nullopt if no such segment exists.
*/
std::optional<std::string> lastPathSegment() const;
};
std::ostream & operator<<(std::ostream & os, const VerbatimURL & url);
+4 -116
View File
@@ -1,7 +1,4 @@
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/base-n.hh"
#include "nix/util/bytes.hh"
#include "nix/util/json-utils.hh"
namespace nix {
@@ -61,7 +58,7 @@ std::string MemorySourceAccessor::readFile(const CanonPath & path)
if (!f)
throw Error("file '%s' does not exist", path);
if (auto * r = std::get_if<File::Regular>(&f->raw))
return std::string{to_str(r->contents)};
return r->contents;
else
throw Error("file '%s' is not a regular file", path);
}
@@ -128,7 +125,7 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path)
throw Error("file '%s' is not a symbolic link", path);
}
SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector<std::byte> && contents)
SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents)
{
// Create root directory automatically if necessary as a convenience.
if (!root && !path.isRoot())
@@ -145,11 +142,6 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector<std::byte>
return SourcePath{ref(shared_from_this()), path};
}
SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string_view contents)
{
return addFile(path, to_owned(as_bytes(contents)));
}
using File = MemorySourceAccessor::File;
void MemorySink::createDirectory(const CanonPath & path)
@@ -198,10 +190,9 @@ void CreateMemoryRegularFile::preallocateContents(uint64_t len)
regularFile.contents.reserve(len);
}
void CreateMemoryRegularFile::operator()(std::string_view data_)
void CreateMemoryRegularFile::operator()(std::string_view data)
{
auto data = as_bytes(data_);
regularFile.contents.insert(regularFile.contents.end(), data.begin(), data.end());
regularFile.contents += data;
}
void MemorySink::createSymlink(const CanonPath & path, const std::string & target)
@@ -231,106 +222,3 @@ ref<SourceAccessor> makeEmptySourceAccessor()
}
} // namespace nix
namespace nlohmann {
using namespace nix;
MemorySourceAccessor::File::Regular adl_serializer<MemorySourceAccessor::File::Regular>::from_json(const json & json)
{
auto & obj = getObject(json);
return MemorySourceAccessor::File::Regular{
.executable = getBoolean(valueAt(obj, "executable")),
.contents = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "contents"))))),
};
}
void adl_serializer<MemorySourceAccessor::File::Regular>::to_json(
json & json, const MemorySourceAccessor::File::Regular & val)
{
json = {
{"executable", val.executable},
{"contents", base64::encode(val.contents)},
};
}
MemorySourceAccessor::File::Directory
adl_serializer<MemorySourceAccessor::File::Directory>::from_json(const json & json)
{
auto & obj = getObject(json);
return MemorySourceAccessor::File::Directory{
.contents = valueAt(obj, "contents"),
};
}
void adl_serializer<MemorySourceAccessor::File::Directory>::to_json(
json & json, const MemorySourceAccessor::File::Directory & val)
{
json = {
{"contents", val.contents},
};
}
MemorySourceAccessor::File::Symlink adl_serializer<MemorySourceAccessor::File::Symlink>::from_json(const json & json)
{
auto & obj = getObject(json);
return MemorySourceAccessor::File::Symlink{
.target = getString(valueAt(obj, "target")),
};
}
void adl_serializer<MemorySourceAccessor::File::Symlink>::to_json(
json & json, const MemorySourceAccessor::File::Symlink & val)
{
json = {
{"target", val.target},
};
}
MemorySourceAccessor::File adl_serializer<MemorySourceAccessor::File>::from_json(const json & json)
{
auto & obj = getObject(json);
auto type = getString(valueAt(obj, "type"));
if (type == "regular")
return static_cast<MemorySourceAccessor::File::Regular>(json);
if (type == "directory")
return static_cast<MemorySourceAccessor::File::Directory>(json);
if (type == "symlink")
return static_cast<MemorySourceAccessor::File::Symlink>(json);
else
throw Error("unknown type of file '%s'", type);
}
void adl_serializer<MemorySourceAccessor::File>::to_json(json & json, const MemorySourceAccessor::File & val)
{
std::visit(
overloaded{
[&](const MemorySourceAccessor::File::Regular & r) {
json = r;
json["type"] = "regular";
},
[&](const MemorySourceAccessor::File::Directory & d) {
json = d;
json["type"] = "directory";
},
[&](const MemorySourceAccessor::File::Symlink & s) {
json = s;
json["type"] = "symlink";
},
},
val.raw);
}
MemorySourceAccessor adl_serializer<MemorySourceAccessor>::from_json(const json & json)
{
MemorySourceAccessor res;
res.root = json;
return res;
}
void adl_serializer<MemorySourceAccessor>::to_json(json & json, const MemorySourceAccessor & val)
{
json = val.root;
}
} // namespace nlohmann
+1
View File
@@ -156,6 +156,7 @@ sources = [ config_priv_h ] + files(
'source-accessor.cc',
'source-path.cc',
'strings.cc',
'subdir-source-accessor.cc',
'suggestions.cc',
'tarfile.cc',
'tee-logger.cc',
+3 -45
View File
@@ -94,8 +94,9 @@ void Source::drainInto(Sink & sink)
{
std::array<char, 8192> buf;
while (true) {
size_t n;
try {
auto n = read(buf.data(), buf.size());
n = read(buf.data(), buf.size());
sink({buf.data(), n});
} catch (EndOfFile &) {
break;
@@ -110,16 +111,6 @@ std::string Source::drain()
return std::move(s.s);
}
void Source::skip(size_t len)
{
std::array<char, 8192> buf;
while (len) {
auto n = read(buf.data(), std::min(len, buf.size()));
assert(n <= len);
len -= n;
}
}
size_t BufferedSource::read(char * data, size_t len)
{
if (!buffer)
@@ -129,7 +120,7 @@ size_t BufferedSource::read(char * data, size_t len)
bufPosIn = readUnbuffered(buffer.get(), bufSize);
/* Copy out the data in the buffer. */
auto n = std::min(len, bufPosIn - bufPosOut);
size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
memcpy(data, buffer.get() + bufPosOut, n);
bufPosOut += n;
if (bufPosIn == bufPosOut)
@@ -200,39 +191,6 @@ bool FdSource::hasData()
}
}
void FdSource::skip(size_t len)
{
/* Discard data in the buffer. */
if (len && buffer && bufPosIn - bufPosOut) {
if (len >= bufPosIn - bufPosOut) {
len -= bufPosIn - bufPosOut;
bufPosIn = bufPosOut = 0;
} else {
bufPosOut += len;
len = 0;
}
}
#ifndef _WIN32
/* If we can, seek forward in the file to skip the rest. */
if (isSeekable && len) {
if (lseek(fd, len, SEEK_CUR) == -1) {
if (errno == ESPIPE)
isSeekable = false;
else
throw SysError("seeking forward in file");
} else {
read += len;
return;
}
}
#endif
/* Otherwise, skip by reading. */
if (len)
BufferedSource::skip(len);
}
size_t StringSource::read(char * data, size_t len)
{
if (pos == s.size())
+59
View File
@@ -0,0 +1,59 @@
#include "nix/util/source-accessor.hh"
namespace nix {
struct SubdirSourceAccessor : SourceAccessor
{
ref<SourceAccessor> parent;
CanonPath subdirectory;
SubdirSourceAccessor(ref<SourceAccessor> && parent, CanonPath && subdirectory)
: parent(std::move(parent))
, subdirectory(std::move(subdirectory))
{
displayPrefix.clear();
}
std::string readFile(const CanonPath & path) override
{
return parent->readFile(subdirectory / path);
}
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
{
return parent->readFile(subdirectory / path, sink, sizeCallback);
}
bool pathExists(const CanonPath & path) override
{
return parent->pathExists(subdirectory / path);
}
std::optional<Stat> maybeLstat(const CanonPath & path) override
{
return parent->maybeLstat(subdirectory / path);
}
DirEntries readDirectory(const CanonPath & path) override
{
return parent->readDirectory(subdirectory / path);
}
std::string readLink(const CanonPath & path) override
{
return parent->readLink(subdirectory / path);
}
std::string showPath(const CanonPath & path) override
{
return displayPrefix + parent->showPath(subdirectory / path) + displaySuffix;
}
};
ref<SourceAccessor> projectSubdirSourceAccessor(ref<SourceAccessor> parent, CanonPath subdirectory)
{
return make_ref<SubdirSourceAccessor>(std::move(parent), std::move(subdirectory));
}
} // namespace nix
-18
View File
@@ -4,7 +4,6 @@
#include "nix/util/split.hh"
#include "nix/util/canon-path.hh"
#include "nix/util/strings-inline.hh"
#include "nix/util/file-system.hh"
#include <boost/url.hpp>
@@ -441,21 +440,4 @@ std::ostream & operator<<(std::ostream & os, const VerbatimURL & url)
return os;
}
std::optional<std::string> VerbatimURL::lastPathSegment() const
{
try {
auto parsedUrl = parsed();
auto segments = parsedUrl.pathSegments(/*skipEmpty=*/true);
if (std::ranges::empty(segments))
return std::nullopt;
return segments.back();
} catch (BadURL &) {
// Fall back to baseNameOf for unparsable URLs
auto name = baseNameOf(to_string());
if (name.empty())
return std::nullopt;
return std::string{name};
}
}
} // namespace nix
-2
View File
@@ -76,8 +76,6 @@ struct CmdCatNar : StoreCommand, MixCat
void run(ref<Store> store) override
{
AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
if (!fd)
throw SysError("opening NAR file '%s'", narPath);
auto source = FdSource{fd.get()};
auto narAccessor = makeNarAccessor(source);
auto listing = listNar(narAccessor, CanonPath::root, true);
+1 -1
View File
@@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand
{
auto json = nlohmann::json::parse(drainFD(STDIN_FILENO));
auto drv = static_cast<Derivation>(json);
auto drv = Derivation::fromJSON(json);
auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun);
+3 -2
View File
@@ -12,7 +12,8 @@ a Nix expression evaluates.
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
`nix derivation add` takes a single derivation in the JSON format.
See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format.
`nix derivation add` takes a single derivation in the following format:
{{#include ../../protocols/json/derivation.md}}
)""
+1 -1
View File
@@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON
if (!drvPath.isDerivation())
continue;
jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath);
jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON();
}
printJSON(jsonRoot);
}
+3 -2
View File
@@ -48,9 +48,10 @@ By default, this command only shows top-level derivations, but with
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
`nix derivation show` outputs a JSON map of [store path]s to derivations in JSON format.
See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format.
`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format:
[store path]: @docroot@/store/store-path.md
{{#include ../../protocols/json/derivation.md}}
)""
+4 -9
View File
@@ -254,15 +254,10 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
drv.args = {store->printStorePath(getEnvShPath)};
/* Remove derivation checks. */
if (drv.structuredAttrs) {
drv.structuredAttrs->structuredAttrs.erase("outputChecks");
} else {
drv.env.erase("allowedReferences");
drv.env.erase("allowedRequisites");
drv.env.erase("disallowedReferences");
drv.env.erase("disallowedRequisites");
}
drv.env.erase("allowedReferences");
drv.env.erase("allowedRequisites");
drv.env.erase("disallowedReferences");
drv.env.erase("disallowedRequisites");
drv.env.erase("name");
/* Rehash and write the derivation. FIXME: would be nice to use
+2 -10
View File
@@ -4,14 +4,6 @@
using namespace nix;
static FdSink getNarSink()
{
auto fd = getStandardOutput();
if (isatty(fd))
throw UsageError("refusing to write NAR to a terminal");
return FdSink(std::move(fd));
}
struct CmdDumpPath : StorePathCommand
{
std::string description() override
@@ -28,7 +20,7 @@ struct CmdDumpPath : StorePathCommand
void run(ref<Store> store, const StorePath & storePath) override
{
auto sink = getNarSink();
FdSink sink(getStandardOutput());
store->narFromPath(storePath, sink);
sink.flush();
}
@@ -59,7 +51,7 @@ struct CmdDumpPath2 : Command
void run() override
{
auto sink = getNarSink();
FdSink sink(getStandardOutput());
dumpPath(path, sink);
sink.flush();
}
+2 -3
View File
@@ -85,9 +85,8 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
if (pathExists(*writeTo))
throw Error("path '%s' already exists", writeTo->string());
std::function<void(Value & v, const PosIdx pos, const std::filesystem::path & path)> recurse;
recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path) {
auto recurse =
[&](this auto & recurse, Value & v, const PosIdx pos, const std::filesystem::path & path) -> void {
state->forceValue(v, pos);
if (v.type() == nString)
// FIXME: disallow strings with contexts?
+1 -2
View File
@@ -38,8 +38,7 @@ struct CmdFlakePrefetchInputs : FlakeCommand
std::atomic<size_t> nrFailed{0};
std::function<void(const Node & node)> visit;
visit = [&](const Node & node) {
auto visit = [&](this auto & visit, const Node & node) -> void {
if (!state_.lock()->done.insert(&node).second)
return;
+1 -3
View File
@@ -269,9 +269,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
std::set<ref<Node>> visited;
std::function<void(const Node & node, const std::string & prefix)> recurse;
recurse = [&](const Node & node, const std::string & prefix) {
auto recurse = [&](this auto & recurse, const Node & node, const std::string & prefix) -> void {
for (const auto & [i, input] : enumerate(node.inputs)) {
bool last = i + 1 == node.inputs.size();
-2
View File
@@ -146,8 +146,6 @@ struct CmdLsNar : Command, MixLs
void run() override
{
AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
if (!fd)
throw SysError("opening NAR file '%s'", narPath);
auto source = FdSource{fd.get()};
auto narAccessor = makeNarAccessor(source);
auto listing = listNar(narAccessor, CanonPath::root, true);
+7 -15
View File
@@ -13,8 +13,6 @@
#include "nix/cmd/misc-store-flags.hh"
#include "nix/util/terminal.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/url.hh"
#include "nix/store/path.hh"
#include "man-pages.hh"
@@ -58,7 +56,7 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url)
std::tuple<StorePath, Hash> prefetchFile(
ref<Store> store,
const VerbatimURL & url,
std::string_view url,
std::optional<std::string> name,
HashAlgorithm hashAlgo,
std::optional<Hash> expectedHash,
@@ -70,15 +68,9 @@ std::tuple<StorePath, Hash> prefetchFile(
/* Figure out a name in the Nix store. */
if (!name) {
name = url.lastPathSegment();
if (!name || name->empty())
throw Error("cannot figure out file name for '%s'", url.to_string());
}
try {
checkName(*name);
} catch (BadStorePathName & e) {
e.addTrace({}, "file name '%s' was extracted from URL '%s'", *name, url.to_string());
throw;
name = baseNameOf(url);
if (name->empty())
throw Error("cannot figure out file name for '%s'", url);
}
std::optional<StorePath> storePath;
@@ -113,14 +105,14 @@ std::tuple<StorePath, Hash> prefetchFile(
FdSink sink(fd.get());
FileTransferRequest req(url);
FileTransferRequest req(VerbatimURL{url});
req.decompress = false;
getFileTransfer()->download(std::move(req), sink);
}
/* Optionally unpack the file. */
if (unpack) {
Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url.to_string()));
Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url));
auto unpacked = (tmpDir.path() / "unpacked").string();
createDirs(unpacked);
unpackTarfile(tmpFile.string(), unpacked);
@@ -136,7 +128,7 @@ std::tuple<StorePath, Hash> prefetchFile(
}
}
Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string()));
Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url));
auto info = store->addToStoreSlow(
*name, PosixSourceAccessor::createAtRoot(tmpFile), method, hashAlgo, {}, expectedHash);
+4 -4
View File
@@ -90,10 +90,10 @@ struct CmdSearch : InstallableValueCommand, MixJSON
uint64_t results = 0;
std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)>
visit;
visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse) {
auto visit = [&](this auto & visit,
eval_cache::AttrCursor & cursor,
const std::vector<Symbol> & attrPath,
bool initialRecurse) -> void {
auto attrPathS = state->symbols.resolve(attrPath);
Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPathS)));
+5 -6
View File
@@ -160,16 +160,15 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions
}
}
struct BailOut
{};
/* Print the subgraph of nodes that have 'dependency' in their
closure (i.e., that have a non-infinite distance to
'dependency'). Print every edge on a path between `package`
and `dependency`. */
std::function<void(Node &, const std::string &, const std::string &)> printNode;
struct BailOut
{};
printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) {
auto printNode =
[&](this auto & printNode, Node & node, const std::string & firstPad, const std::string & tailPad) -> void {
assert(node.dist != inf);
if (precise) {
logger->cout(
-18
View File
@@ -18,21 +18,6 @@ cat <<EOF >"$TEST_HOME/flake.nix"
outputs = [ "out" "dev" ];
meta.outputsToInstall = [ "out" ];
buildCommand = "";
# ensure we're stripping these from the environment derivation
disallowedReferences = [ "out" ];
disallowedRequisites = [ "out" ];
};
packages.$system.hello-structured = (import ./config.nix).mkDerivation {
__structuredAttrs = true;
name = "hello";
outputs = [ "out" "dev" ];
meta.outputsToInstall = [ "out" ];
buildCommand = "";
# ensure we're stripping these from the environment derivation
outputChecks.out = {
disallowedReferences = [ "out" ];
disallowedRequisites = [ "out" ];
};
};
};
}
@@ -157,7 +142,4 @@ echo "\$SHELL"
EOF
)" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
# Test whether `nix develop` works with `__structuredAttrs`
[[ -z "$(nix develop --no-write-lock-file .#hello-structured </dev/null)" ]]
clearStore
-7
View File
@@ -25,13 +25,6 @@ import $testDir/undefined-variable.nix
TODO_NixOS
# FIXME: repl tests fail on systems with stack limits
stack_ulimit="$(ulimit -Hs)"
stack_required="$((64 * 1024 * 1024))"
if [[ "$stack_ulimit" != "unlimited" ]]; then
((stack_ulimit < stack_required)) && skipTest "repl tests cannot run on systems with stack size <$stack_required ($stack_ulimit)"
fi
testRepl () {
local nixArgs
nixArgs=("$@")
+39 -166
View File
@@ -83,11 +83,9 @@ in
ENDPOINT = 'http://server:9000'
REGION = 'eu-west-1'
PKGS = {
'A': '${pkgA}',
'B': '${pkgB}',
'C': '${pkgC}',
}
PKG_A = '${pkgA}'
PKG_B = '${pkgB}'
PKG_C = '${pkgC}'
ENV_WITH_CREDS = f"AWS_ACCESS_KEY_ID={ACCESS_KEY} AWS_SECRET_ACCESS_KEY={SECRET_KEY}"
@@ -103,6 +101,10 @@ in
bucket_and_path = f"{bucket}{path}" if path else bucket
return f"s3://{bucket_and_path}?{query}"
def make_http_url(path):
"""Build HTTP URL for direct S3 access"""
return f"{ENDPOINT}/{path}"
def get_package_hash(pkg_path):
"""Extract store hash from package path"""
return pkg_path.split("/")[-1].split("-")[0]
@@ -131,49 +133,26 @@ in
print(output)
raise Exception(f"{error_msg}: expected {expected}, got {actual}")
def verify_packages_in_store(machine, pkg_paths, should_exist=True):
"""
Verify whether packages exist in the store.
Args:
machine: The machine to check on
pkg_paths: List of package paths to check (or single path)
should_exist: If True, verify packages exist; if False, verify they don't
"""
paths = [pkg_paths] if isinstance(pkg_paths, str) else pkg_paths
for pkg in paths:
if should_exist:
machine.succeed(f"nix path-info {pkg}")
else:
machine.fail(f"nix path-info {pkg}")
def setup_s3(populate_bucket=[], public=False):
def with_test_bucket(populate_with=[]):
"""
Decorator that creates/destroys a unique bucket for each test.
Optionally pre-populates bucket with specified packages.
Cleans up client store after test completion.
Args:
populate_bucket: List of packages to upload before test runs
public: If True, make the bucket publicly accessible
populate_with: List of packages to upload before test runs
"""
def decorator(test_func):
def wrapper():
bucket = str(uuid.uuid4())
server.succeed(f"mc mb minio/{bucket}")
if public:
server.succeed(f"mc anonymous set download minio/{bucket}")
try:
if populate_bucket:
if populate_with:
store_url = make_s3_url(bucket)
for pkg in populate_bucket:
for pkg in populate_with:
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {pkg}")
test_func(bucket)
finally:
server.succeed(f"mc rb --force minio/{bucket}")
# Clean up client store - only delete if path exists
for pkg in PKGS.values():
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
return wrapper
return decorator
@@ -181,7 +160,7 @@ in
# Test Functions
# ============================================================================
@setup_s3()
@with_test_bucket()
def test_credential_caching(bucket):
"""Verify credential providers are cached and reused"""
print("\n=== Testing Credential Caching ===")
@@ -189,7 +168,7 @@ in
store_url = make_s3_url(bucket)
output = server.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' "
f"{PKGS['A']} {PKGS['B']} {PKGS['C']} 2>&1"
f"{PKG_A} {PKG_B} {PKG_C} 2>&1"
)
assert_count(
@@ -201,7 +180,7 @@ in
print("✓ Credential provider created once and cached")
@setup_s3(populate_bucket=[PKGS['A']])
@with_test_bucket(populate_with=[PKG_A])
def test_fetchurl_basic(bucket):
"""Test builtins.fetchurl works with s3:// URLs"""
print("\n=== Testing builtins.fetchurl ===")
@@ -217,13 +196,13 @@ in
print("✓ builtins.fetchurl works with s3:// URLs")
@setup_s3()
@with_test_bucket()
def test_error_message_formatting(bucket):
"""Verify error messages display URLs correctly"""
print("\n=== Testing Error Message Formatting ===")
nonexistent_url = make_s3_url(bucket, path="/foo-that-does-not-exist")
expected_http_url = f"{ENDPOINT}/{bucket}/foo-that-does-not-exist"
expected_http_url = make_http_url(f"{bucket}/foo-that-does-not-exist")
error_msg = client.fail(
f"{ENV_WITH_CREDS} nix eval --impure --expr "
@@ -237,7 +216,7 @@ in
print("✓ Error messages format URLs correctly")
@setup_s3(populate_bucket=[PKGS['A']])
@with_test_bucket(populate_with=[PKG_A])
def test_fork_credential_preresolution(bucket):
"""Test credential pre-resolution in forked processes"""
print("\n=== Testing Fork Credential Pre-resolution ===")
@@ -267,7 +246,7 @@ in
""".format(id=test_id, url=test_url, hash=cache_info_hash)
output = client.succeed(
f"{ENV_WITH_CREDS} nix build --debug --impure --no-link --expr '{fetchurl_expr}' 2>&1"
f"{ENV_WITH_CREDS} nix build --debug --impure --expr '{fetchurl_expr}' 2>&1"
)
# Verify fork behavior
@@ -317,7 +296,7 @@ in
print(" ✓ Child uses pre-resolved credentials (no new providers)")
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B'], PKGS['C']])
@with_test_bucket(populate_with=[PKG_A, PKG_B, PKG_C])
def test_store_operations(bucket):
"""Test nix store info and copy operations"""
print("\n=== Testing Store Operations ===")
@@ -337,11 +316,11 @@ in
print(f" ✓ Store URL: {store_info['url']}")
# Test copy from store
verify_packages_in_store(client, PKGS['A'], should_exist=False)
client.fail(f"nix path-info {PKG_A}")
output = client.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs "
f"--from '{store_url}' {PKGS['A']} {PKGS['B']} {PKGS['C']} 2>&1"
f"--from '{store_url}' {PKG_A} {PKG_B} {PKG_C} 2>&1"
)
assert_count(
@@ -351,46 +330,12 @@ in
"Client credential provider caching failed"
)
verify_packages_in_store(client, [PKGS['A'], PKGS['B'], PKGS['C']])
client.succeed(f"nix path-info {PKG_A}")
print(" ✓ nix copy works")
print(" ✓ Credentials cached on client")
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B']], public=True)
def test_public_bucket_operations(bucket):
"""Test store operations on public bucket without credentials"""
print("\n=== Testing Public Bucket Operations ===")
store_url = make_s3_url(bucket)
# Verify store info works without credentials
client.succeed(f"nix store info --store '{store_url}' >&2")
print(" ✓ nix store info works without credentials")
# Get and validate store info JSON
info_json = client.succeed(f"nix store info --json --store '{store_url}'")
store_info = json.loads(info_json)
if not store_info.get("url"):
raise Exception("Store should have a URL")
print(f" ✓ Store URL: {store_info['url']}")
# Verify packages are not yet in client store
verify_packages_in_store(client, [PKGS['A'], PKGS['B']], should_exist=False)
# Test copy from public bucket without credentials
client.succeed(
f"nix copy --debug --no-check-sigs "
f"--from '{store_url}' {PKGS['A']} {PKGS['B']} 2>&1"
)
# Verify packages were copied successfully
verify_packages_in_store(client, [PKGS['A'], PKGS['B']])
print(" ✓ nix copy from public bucket works without credentials")
@setup_s3(populate_bucket=[PKGS['A']])
@with_test_bucket(populate_with=[PKG_A])
def test_url_format_variations(bucket):
"""Test different S3 URL parameter combinations"""
print("\n=== Testing URL Format Variations ===")
@@ -405,7 +350,7 @@ in
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2")
print(" ✓ Parameter order: endpoint before region works")
@setup_s3(populate_bucket=[PKGS['A']])
@with_test_bucket(populate_with=[PKG_A])
def test_concurrent_fetches(bucket):
"""Validate thread safety with concurrent S3 operations"""
print("\n=== Testing Concurrent Fetches ===")
@@ -441,12 +386,12 @@ in
try:
output = client.succeed(
f"{ENV_WITH_CREDS} nix build --debug --impure --no-link "
f"{ENV_WITH_CREDS} nix build --debug --impure "
f"--expr '{concurrent_expr}' --max-jobs 5 2>&1"
)
except:
output = client.fail(
f"{ENV_WITH_CREDS} nix build --debug --impure --no-link "
f"{ENV_WITH_CREDS} nix build --debug --impure "
f"--expr '{concurrent_expr}' --max-jobs 5 2>&1"
)
@@ -467,33 +412,26 @@ in
f"Expected 5 FileTransfer instances for 5 concurrent fetches, got {transfers_created}"
)
if providers_created != 1:
print("Debug output:")
print(output)
raise Exception(
f"Expected 1 credential provider for concurrent fetches, got {providers_created}"
)
@setup_s3()
@with_test_bucket()
def test_compression_narinfo_gzip(bucket):
"""Test narinfo compression with gzip"""
print("\n=== Testing Compression: narinfo (gzip) ===")
store_url = make_s3_url(bucket, **{'narinfo-compression': 'gzip'})
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['B']}")
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_B}")
pkg_hash = get_package_hash(PKGS['B'])
pkg_hash = get_package_hash(PKG_B)
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip")
print(" ✓ .narinfo has Content-Encoding: gzip")
# Verify client can download and decompress
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}")
verify_packages_in_store(client, PKGS['B'])
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_B}")
client.succeed(f"nix path-info {PKG_B}")
print(" ✓ Client decompressed .narinfo successfully")
@setup_s3()
@with_test_bucket()
def test_compression_mixed(bucket):
"""Test mixed compression (narinfo=xz, ls=gzip)"""
print("\n=== Testing Compression: mixed (narinfo=xz, ls=gzip) ===")
@@ -503,9 +441,9 @@ in
**{'narinfo-compression': 'xz', 'write-nar-listing': 'true', 'ls-compression': 'gzip'}
)
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['C']}")
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_C}")
pkg_hash = get_package_hash(PKGS['C'])
pkg_hash = get_package_hash(PKG_C)
# Verify .narinfo has xz compression
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz")
@@ -516,87 +454,24 @@ in
print(" ✓ .ls has Content-Encoding: gzip")
# Verify client can download with mixed compression
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}")
verify_packages_in_store(client, PKGS['C'])
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_C}")
client.succeed(f"nix path-info {PKG_C}")
print(" ✓ Client downloaded package with mixed compression")
@setup_s3()
@with_test_bucket()
def test_compression_disabled(bucket):
"""Verify no compression by default"""
print("\n=== Testing Compression: disabled (default) ===")
store_url = make_s3_url(bucket)
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']}")
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_A}")
pkg_hash = get_package_hash(PKGS['A'])
pkg_hash = get_package_hash(PKG_A)
verify_no_compression(server, bucket, f"{pkg_hash}.narinfo")
print(" ✓ No compression applied by default")
@setup_s3()
def test_nix_prefetch_url(bucket):
"""Test that nix-prefetch-url retrieves actual file content from S3, not empty files (issue #8862)"""
print("\n=== Testing nix-prefetch-url S3 Content Retrieval (issue #8862) ===")
# Create a test file with known content
test_content = "This is test content to verify S3 downloads work correctly!\n"
test_file_size = len(test_content)
server.succeed(f"echo -n '{test_content}' > /tmp/test-file.txt")
# Upload to S3
server.succeed(f"mc cp /tmp/test-file.txt minio/{bucket}/test-file.txt")
# Calculate expected hash
expected_hash = server.succeed(
"nix hash file --type sha256 --base32 /tmp/test-file.txt"
).strip()
print(f" ✓ Uploaded test file to S3 ({test_file_size} bytes)")
# Use nix-prefetch-url to download from S3
s3_url = make_s3_url(bucket, path="/test-file.txt")
prefetch_output = client.succeed(
f"{ENV_WITH_CREDS} nix-prefetch-url --print-path '{s3_url}'"
)
# Extract hash and store path
# With --print-path, output is: <hash>\n<store-path>
lines = prefetch_output.strip().split('\n')
prefetch_hash = lines[0] # First line is the hash
store_path = lines[1] # Second line is the store path
# Verify hash matches
if prefetch_hash != expected_hash:
raise Exception(
f"Hash mismatch: expected {expected_hash}, got {prefetch_hash}"
)
print(" ✓ nix-prefetch-url completed with correct hash")
# Verify the downloaded file is NOT empty (the bug in #8862)
file_size = int(client.succeed(f"stat -c %s {store_path}").strip())
if file_size == 0:
raise Exception("Downloaded file is EMPTY - issue #8862 regression detected!")
if file_size != test_file_size:
raise Exception(
f"File size mismatch: expected {test_file_size}, got {file_size}"
)
print(f" ✓ File has correct size ({file_size} bytes, not empty)")
# Verify actual content matches by comparing hashes instead of printing entire file
downloaded_hash = client.succeed(f"nix hash file --type sha256 --base32 {store_path}").strip()
if downloaded_hash != expected_hash:
raise Exception(f"Content hash mismatch: expected {expected_hash}, got {downloaded_hash}")
print(" ✓ File content verified correct (hash matches)")
# ============================================================================
# Main Test Execution
# ============================================================================
@@ -619,13 +494,11 @@ in
test_error_message_formatting()
test_fork_credential_preresolution()
test_store_operations()
test_public_bucket_operations()
test_url_format_variations()
test_concurrent_fetches()
test_compression_narinfo_gzip()
test_compression_mixed()
test_compression_disabled()
test_nix_prefetch_url()
print("\n" + "="*80)
print("✓ All S3 Binary Cache Store Tests Passed!")