Compare commits
52
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a4b62a646a | ||
|
|
ef8218f2e3 | ||
|
|
ada008a795 | ||
|
|
2a2bb8330d | ||
|
|
645794b458 | ||
|
|
1121f0d8ec | ||
|
|
6420879728 | ||
|
|
67f5cb97a3 | ||
|
|
5e7ee808de | ||
|
|
270f20a505 | ||
|
|
1b1d7e3047 | ||
|
|
0f28c76a44 | ||
|
|
e3b3f05e5d | ||
|
|
f05d240222 | ||
|
|
22c73868c3 | ||
|
|
a91b787524 | ||
|
|
ddf7de0a76 | ||
|
|
1fabed18b6 | ||
|
|
6c9083db2c | ||
|
|
c663f7ec79 | ||
|
|
d0fb03c35d | ||
|
|
c847cd87f1 | ||
|
|
dbbdae926b | ||
|
|
3c03050cd6 | ||
|
|
e33cd5aa38 | ||
|
|
d9c808f8a7 | ||
|
|
55ea3d3476 | ||
|
|
7d0c06f921 | ||
|
|
5b4bd5bcb8 | ||
|
|
4ae6c65bc5 | ||
|
|
4f19e63a8f | ||
|
|
f88c3055f8 | ||
|
|
9058d90ab2 | ||
|
|
c1a15d1a26 | ||
|
|
22f4cccc71 | ||
|
|
b56e456b0d | ||
|
|
c92ba4b9b7 | ||
|
|
67bffa19a5 | ||
|
|
daa7e0d2e9 | ||
|
|
109f6449cc | ||
|
|
ad2360c59f | ||
|
|
20c7c551bf | ||
|
|
e78e6ca4f4 | ||
|
|
e34063cf21 | ||
|
|
e457ea7688 | ||
|
|
ffbc33fec6 | ||
|
|
a80fc252e8 | ||
|
|
bcd5a9d05c | ||
|
|
01b001d5ba | ||
|
|
1177d65094 | ||
|
|
a2c6f38e1f | ||
|
|
1c02dd5b9c |
No files matched your search
@@ -116,6 +116,7 @@ rec {
|
||||
) nixComponentsInstrumented)
|
||||
// lib.optionalAttrs (pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform) {
|
||||
"${componentTestsPrefix}nix-functional-tests" = nixComponentsInstrumented.nix-functional-tests;
|
||||
"${componentTestsPrefix}nix-json-schema-checks" = nixComponentsInstrumented.nix-json-schema-checks;
|
||||
};
|
||||
|
||||
codeCoverage =
|
||||
|
||||
@@ -88,7 +88,7 @@ manual = custom_target(
|
||||
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
|
||||
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
|
||||
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
|
||||
@4@ -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
|
||||
@4@ -r -L --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
|
||||
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
|
||||
rm -rf @2@/manual
|
||||
mv @2@/html @2@/manual
|
||||
@@ -115,6 +115,7 @@ manual = custom_target(
|
||||
builtins_md,
|
||||
rl_next_generated,
|
||||
summary_rl_next,
|
||||
json_schema_generated_files,
|
||||
nix_input,
|
||||
],
|
||||
output : [
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
rsync,
|
||||
nix-cli,
|
||||
changelog-d,
|
||||
json-schema-for-humans,
|
||||
officialRelease,
|
||||
|
||||
# Configuration Options
|
||||
@@ -32,6 +33,9 @@ mkMesonDerivation (finalAttrs: {
|
||||
fileset.difference
|
||||
(fileset.unions [
|
||||
../../.version
|
||||
# For example JSON
|
||||
../../src/libutil-tests/data/memory-source-accessor
|
||||
../../src/libutil-tests/data/hash
|
||||
# Too many different types of files to filter for now
|
||||
../../doc/manual
|
||||
./.
|
||||
@@ -55,6 +59,7 @@ mkMesonDerivation (finalAttrs: {
|
||||
jq
|
||||
python3
|
||||
rsync
|
||||
json-schema-for-humans
|
||||
changelog-d
|
||||
]
|
||||
++ lib.optionals (!officialRelease) [
|
||||
|
||||
@@ -117,6 +117,8 @@
|
||||
- [Architecture and Design](architecture/architecture.md)
|
||||
- [Formats and Protocols](protocols/index.md)
|
||||
- [JSON Formats](protocols/json/index.md)
|
||||
- [File System Object](protocols/json/file-system-object.md)
|
||||
- [Hash](protocols/json/hash.md)
|
||||
- [Store Object Info](protocols/json/store-object-info.md)
|
||||
- [Derivation](protocols/json/derivation.md)
|
||||
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# Process JSON schema documentation
|
||||
subdir('protocols')
|
||||
|
||||
summary_rl_next = custom_target(
|
||||
command : [
|
||||
bash,
|
||||
|
||||
@@ -1,120 +1,7 @@
|
||||
# Derivation JSON Format
|
||||
{{#include derivation-v3-fixed.md}}
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
> This JSON format is currently
|
||||
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
|
||||
> and subject to change.
|
||||
<!--
|
||||
## Raw Schema
|
||||
|
||||
The JSON serialization of a
|
||||
[derivations](@docroot@/glossary.md#gloss-store-derivation)
|
||||
is a JSON object with the following fields:
|
||||
|
||||
* `name`:
|
||||
The name of the derivation.
|
||||
This is used when calculating the store paths of the derivation's outputs.
|
||||
|
||||
* `version`:
|
||||
Must be `3`.
|
||||
This is a guard that allows us to continue evolving this format.
|
||||
The choice of `3` is fairly arbitrary, but corresponds to this informal version:
|
||||
|
||||
- Version 0: A-Term format
|
||||
|
||||
- Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
|
||||
|
||||
- Version 2: Separate `method` and `hashAlgo` fields in output specs
|
||||
|
||||
- Version 3: Drop store dir from store paths, just include base name.
|
||||
|
||||
Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
|
||||
|
||||
* `outputs`:
|
||||
Information about the output paths of the derivation.
|
||||
This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
|
||||
|
||||
* `path`:
|
||||
The output path, if it is known in advanced.
|
||||
Otherwise, `null`.
|
||||
|
||||
|
||||
* `method`:
|
||||
For an output which will be [content addressed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
|
||||
Valid method strings are:
|
||||
|
||||
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
|
||||
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
|
||||
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
|
||||
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
|
||||
|
||||
Otherwise, `null`.
|
||||
|
||||
* `hashAlgo`:
|
||||
For an output which will be [content addressed], the name of the hash algorithm used.
|
||||
Valid algorithm strings are:
|
||||
|
||||
- `blake3`
|
||||
- `md5`
|
||||
- `sha1`
|
||||
- `sha256`
|
||||
- `sha512`
|
||||
|
||||
* `hash`:
|
||||
For fixed-output derivations, the expected content hash in base-16.
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```json
|
||||
> "outputs": {
|
||||
> "out": {
|
||||
> "method": "nar",
|
||||
> "hashAlgo": "sha256",
|
||||
> "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
|
||||
> }
|
||||
> }
|
||||
> ```
|
||||
|
||||
* `inputSrcs`:
|
||||
A list of store paths on which this derivation depends.
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```json
|
||||
> "inputSrcs": [
|
||||
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
|
||||
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
|
||||
> ]
|
||||
> ```
|
||||
|
||||
* `inputDrvs`:
|
||||
A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations.
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```json
|
||||
> "inputDrvs": {
|
||||
> "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
|
||||
> "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
|
||||
> }
|
||||
> ```
|
||||
|
||||
specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
|
||||
|
||||
* `system`:
|
||||
The system type on which this derivation is to be built
|
||||
(e.g. `x86_64-linux`).
|
||||
|
||||
* `builder`:
|
||||
The absolute path of the program to be executed to run the build.
|
||||
Typically this is the `bash` shell
|
||||
(e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
|
||||
|
||||
* `args`:
|
||||
The command-line arguments passed to the `builder`.
|
||||
|
||||
* `env`:
|
||||
The environment passed to the `builder`.
|
||||
|
||||
* `structuredAttrs`:
|
||||
[Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
|
||||
Structured attributes are JSON, and thus embedded as-is.
|
||||
[JSON Schema for Derivation v3](schema/derivation-v3.json)
|
||||
-->
|
||||
@@ -0,0 +1,21 @@
|
||||
{{#include file-system-object-v1-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### Simple
|
||||
|
||||
```json
|
||||
{{#include schema/file-system-object-v1/simple.json}}
|
||||
```
|
||||
|
||||
### Complex
|
||||
|
||||
```json
|
||||
{{#include schema/file-system-object-v1/complex.json}}
|
||||
```
|
||||
|
||||
<!--
|
||||
## Raw Schema
|
||||
|
||||
[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
|
||||
-->
|
||||
@@ -0,0 +1,15 @@
|
||||
# For some reason, backticks in the JSON schema are being escaped rather
|
||||
# than being kept as intentional code spans. This removes all backtick
|
||||
# escaping, which is an ugly solution, but one that is fine, because we
|
||||
# are not using backticks for any other purpose.
|
||||
s/\\`/`/g
|
||||
|
||||
# The way that semi-external references are rendered (i.e. ones to
|
||||
# sibling schema files, as opposed to separate website ones, is not nice
|
||||
# for humans. Replace it with a nice relative link within the manual
|
||||
# instead.
|
||||
#
|
||||
# As we have more such relative links, more replacements of this nature
|
||||
# should appear below.
|
||||
s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
|
||||
s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
|
||||
@@ -0,0 +1,33 @@
|
||||
{{#include hash-v1-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### SHA-256 with Base64 encoding
|
||||
|
||||
```json
|
||||
{{#include schema/hash-v1/sha256-base64.json}}
|
||||
```
|
||||
|
||||
### SHA-256 with Base16 (hexadecimal) encoding
|
||||
|
||||
```json
|
||||
{{#include schema/hash-v1/sha256-base16.json}}
|
||||
```
|
||||
|
||||
### SHA-256 with Nix32 encoding
|
||||
|
||||
```json
|
||||
{{#include schema/hash-v1/sha256-nix32.json}}
|
||||
```
|
||||
|
||||
### BLAKE3 with Base64 encoding
|
||||
|
||||
```json
|
||||
{{#include schema/hash-v1/blake3-base64.json}}
|
||||
```
|
||||
|
||||
<!--
|
||||
## Raw Schema
|
||||
|
||||
[JSON Schema for Hash v1](schema/hash-v1.json)
|
||||
-->
|
||||
@@ -0,0 +1,17 @@
|
||||
# Configuration file for json-schema-for-humans
|
||||
#
|
||||
# https://github.com/coveooss/json-schema-for-humans/blob/main/docs/examples/examples_md_default/Configuration.md
|
||||
|
||||
template_name: md
|
||||
show_toc: true
|
||||
# impure timestamp and distracting
|
||||
with_footer: false
|
||||
recursive_detection_depth: 3
|
||||
show_breadcrumbs: false
|
||||
description_is_markdown: true
|
||||
template_md_options:
|
||||
properties_table_columns:
|
||||
- Property
|
||||
- Type
|
||||
- Pattern
|
||||
- Title/Description
|
||||
@@ -0,0 +1,75 @@
|
||||
# Tests in: ../../../../src/json-schema-checks
|
||||
|
||||
fs = import('fs')
|
||||
|
||||
# Find json-schema-for-humans if available
|
||||
json_schema_for_humans = find_program('generate-schema-doc', required : false)
|
||||
|
||||
# Configuration for json-schema-for-humans
|
||||
json_schema_config = files('json-schema-for-humans-config.yaml')
|
||||
|
||||
schemas = [
|
||||
'file-system-object-v1',
|
||||
'hash-v1',
|
||||
'derivation-v3',
|
||||
]
|
||||
|
||||
schema_files = files()
|
||||
foreach schema_name : schemas
|
||||
schema_files += files('schema' / schema_name + '.yaml')
|
||||
endforeach
|
||||
|
||||
|
||||
schema_outputs = []
|
||||
foreach schema_name : schemas
|
||||
schema_outputs += schema_name + '.md'
|
||||
endforeach
|
||||
|
||||
json_schema_generated_files = []
|
||||
|
||||
# Generate markdown documentation from JSON schema
|
||||
# Note: output must be just a filename, not a path
|
||||
gen_file = custom_target(
|
||||
schema_name + '-schema-docs.tmp',
|
||||
command : [
|
||||
json_schema_for_humans,
|
||||
'--config-file',
|
||||
json_schema_config,
|
||||
meson.current_source_dir() / 'schema',
|
||||
meson.current_build_dir(),
|
||||
],
|
||||
input : schema_files + [
|
||||
json_schema_config,
|
||||
],
|
||||
output : schema_outputs,
|
||||
capture : false,
|
||||
build_by_default : true,
|
||||
)
|
||||
|
||||
idx = 0
|
||||
if json_schema_for_humans.found()
|
||||
foreach schema_name : schemas
|
||||
#schema_file = 'schema' / schema_name + '.yaml'
|
||||
|
||||
# There is one so-so hack, and one horrible hack being done here.
|
||||
sedded_file = custom_target(
|
||||
schema_name + '-schema-docs',
|
||||
command : [
|
||||
'sed',
|
||||
'-f',
|
||||
# Out of line to avoid https://github.com/mesonbuild/meson/issues/1564
|
||||
files('fixup-json-schema-generated-doc.sed'),
|
||||
'@INPUT@',
|
||||
],
|
||||
capture : true,
|
||||
input : gen_file[idx],
|
||||
output : schema_name + '-fixed.md',
|
||||
)
|
||||
idx += 1
|
||||
json_schema_generated_files += [ sedded_file ]
|
||||
endforeach
|
||||
else
|
||||
warning(
|
||||
'json-schema-for-humans not found, skipping JSON schema documentation generation',
|
||||
)
|
||||
endif
|
||||
@@ -0,0 +1,178 @@
|
||||
"$schema": http://json-schema.org/draft-04/schema#
|
||||
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json
|
||||
title: Derivation
|
||||
description: |
|
||||
Experimental JSON representation of a Nix derivation (version 3).
|
||||
|
||||
This schema describes the JSON representation of Nix's `Derivation` type.
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
> This JSON format is currently
|
||||
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
|
||||
> and subject to change.
|
||||
|
||||
type: object
|
||||
required:
|
||||
- name
|
||||
- version
|
||||
- outputs
|
||||
- inputSrcs
|
||||
- inputDrvs
|
||||
- system
|
||||
- builder
|
||||
- args
|
||||
- env
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
title: Derivation name
|
||||
description: |
|
||||
The name of the derivation.
|
||||
Used when calculating store paths for the derivation’s outputs.
|
||||
|
||||
version:
|
||||
const: 3
|
||||
title: Format version (must be 3)
|
||||
description: |
|
||||
Must be `3`.
|
||||
This is a guard that allows us to continue evolving this format.
|
||||
The choice of `3` is fairly arbitrary, but corresponds to this informal version:
|
||||
|
||||
- Version 0: A-Term format
|
||||
|
||||
- Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format.
|
||||
|
||||
- Version 2: Separate `method` and `hashAlgo` fields in output specs
|
||||
|
||||
- Version 3: Drop store dir from store paths, just include base name.
|
||||
|
||||
Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change.
|
||||
|
||||
outputs:
|
||||
type: object
|
||||
title: Output specifications
|
||||
description: |
|
||||
Information about the output paths of the derivation.
|
||||
This is a JSON object with one member per output, where the key is the output name and the value is a JSON object as described.
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```json
|
||||
> "outputs": {
|
||||
> "out": {
|
||||
> "method": "nar",
|
||||
> "hashAlgo": "sha256",
|
||||
> "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
|
||||
> }
|
||||
> }
|
||||
> ```
|
||||
additionalProperties:
|
||||
"$ref": "#/$defs/output"
|
||||
|
||||
inputSrcs:
|
||||
type: array
|
||||
title: Input source paths
|
||||
description: |
|
||||
List of store paths on which this derivation depends.
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```json
|
||||
> "inputSrcs": [
|
||||
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
|
||||
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
|
||||
> ]
|
||||
> ```
|
||||
items:
|
||||
type: string
|
||||
|
||||
inputDrvs:
|
||||
type: object
|
||||
title: Input derivations
|
||||
description: |
|
||||
Mapping of derivation paths to lists of output names they provide.
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```json
|
||||
> "inputDrvs": {
|
||||
> "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
|
||||
> "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
|
||||
> }
|
||||
> ```
|
||||
>
|
||||
> specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
|
||||
|
||||
system:
|
||||
type: string
|
||||
title: Build system type
|
||||
description: |
|
||||
The system type on which this derivation is to be built
|
||||
(e.g. `x86_64-linux`).
|
||||
|
||||
builder:
|
||||
type: string
|
||||
title: Build program path
|
||||
description: |
|
||||
Absolute path of the program used to perform the build.
|
||||
Typically this is the `bash` shell
|
||||
(e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
|
||||
|
||||
args:
|
||||
type: array
|
||||
title: Builder arguments
|
||||
description: |
|
||||
Command-line arguments passed to the `builder`.
|
||||
items:
|
||||
type: string
|
||||
|
||||
env:
|
||||
type: object
|
||||
title: Environment variables
|
||||
description: |
|
||||
Environment variables passed to the `builder`.
|
||||
additionalProperties:
|
||||
type: string
|
||||
|
||||
structuredAttrs:
|
||||
title: Structured attributes
|
||||
description: |
|
||||
[Structured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
|
||||
Structured attributes are JSON, and thus embedded as-is.
|
||||
type: object
|
||||
additionalProperties: true
|
||||
|
||||
"$defs":
|
||||
output:
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
title: Output path
|
||||
description: |
|
||||
The output path, if known in advance.
|
||||
|
||||
method:
|
||||
type: string
|
||||
title: Content addressing method
|
||||
enum: [flat, nar, text, git]
|
||||
description: |
|
||||
For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
|
||||
|
||||
Valid method strings are:
|
||||
|
||||
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
|
||||
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
|
||||
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
|
||||
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
|
||||
|
||||
hashAlgo:
|
||||
title: Hash algorithm
|
||||
"$ref": "./hash-v1.yaml#/$defs/algorithm"
|
||||
|
||||
hash:
|
||||
type: string
|
||||
title: Expected hash value
|
||||
description: |
|
||||
For fixed-output derivations, the expected content hash in base-16.
|
||||
@@ -0,0 +1 @@
|
||||
../../../../../../src/libutil-tests/data/memory-source-accessor
|
||||
@@ -0,0 +1,65 @@
|
||||
"$schema": http://json-schema.org/draft-04/schema#
|
||||
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
|
||||
title: File System Object
|
||||
description: |
|
||||
This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
|
||||
|
||||
The schema is recursive because file system objects contain other file system objects.
|
||||
type: object
|
||||
required: ["type"]
|
||||
properties:
|
||||
type:
|
||||
type: string
|
||||
enum: ["regular", "symlink", "directory"]
|
||||
|
||||
# Enforce conditional structure based on `type`
|
||||
anyOf:
|
||||
- $ref: "#/$defs/regular"
|
||||
required: ["type", "contents"]
|
||||
|
||||
- $ref: "#/$defs/symlink"
|
||||
required: ["type", "target"]
|
||||
|
||||
- $ref: "#/$defs/directory"
|
||||
required: ["type", "contents"]
|
||||
|
||||
"$defs":
|
||||
regular:
|
||||
title: Regular File
|
||||
required: ["contents"]
|
||||
properties:
|
||||
type:
|
||||
const: "regular"
|
||||
contents:
|
||||
type: string
|
||||
description: Base64-encoded file contents
|
||||
executable:
|
||||
type: boolean
|
||||
description: Whether the file is executable.
|
||||
default: false
|
||||
additionalProperties: false
|
||||
|
||||
symlink:
|
||||
title: Symbolic Link
|
||||
required: ["target"]
|
||||
properties:
|
||||
type:
|
||||
const: "symlink"
|
||||
target:
|
||||
type: string
|
||||
description: Target path of the symlink.
|
||||
additionalProperties: false
|
||||
|
||||
directory:
|
||||
title: Directory
|
||||
required: ["contents"]
|
||||
properties:
|
||||
type:
|
||||
const: "directory"
|
||||
contents:
|
||||
type: object
|
||||
description: |
|
||||
Map of names to nested file system objects (for type=directory)
|
||||
additionalProperties:
|
||||
$ref: "#"
|
||||
additionalProperties: false
|
||||
@@ -0,0 +1 @@
|
||||
../../../../../../src/libutil-tests/data/hash/
|
||||
@@ -0,0 +1,54 @@
|
||||
"$schema": http://json-schema.org/draft-04/schema#
|
||||
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json
|
||||
title: Hash
|
||||
description: |
|
||||
A cryptographic hash value used throughout Nix for content addressing and integrity verification.
|
||||
|
||||
This schema describes the JSON representation of Nix's `Hash` type.
|
||||
type: object
|
||||
properties:
|
||||
algorithm:
|
||||
"$ref": "#/$defs/algorithm"
|
||||
format:
|
||||
type: string
|
||||
enum:
|
||||
- base64
|
||||
- nix32
|
||||
- base16
|
||||
- sri
|
||||
title: Hash format
|
||||
description: |
|
||||
The encoding format of the hash value.
|
||||
|
||||
- `base64` uses standard Base64 encoding [RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4)
|
||||
- `nix32` is Nix-specific base-32 encoding
|
||||
- `base16` is lowercase hexadecimal
|
||||
- `sri` is the [Subresource Integrity format](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity).
|
||||
hash:
|
||||
type: string
|
||||
title: Hash
|
||||
description: |
|
||||
The encoded hash value, itself.
|
||||
|
||||
It is specified in the format specified by the `format` field.
|
||||
It must be the right length for the hash algorithm specified in the `algorithm` field, also.
|
||||
The hash value does not include any algorithm prefix.
|
||||
required:
|
||||
- algorithm
|
||||
- format
|
||||
- hash
|
||||
additionalProperties: false
|
||||
"$defs":
|
||||
algorithm:
|
||||
type: string
|
||||
enum:
|
||||
- blake3
|
||||
- md5
|
||||
- sha1
|
||||
- sha256
|
||||
- sha512
|
||||
title: Hash algorithm
|
||||
description: |
|
||||
The hash algorithm used to compute the hash value.
|
||||
|
||||
`blake3` is currently experimental and requires the [`blake-hashing`](@docroot@/development/experimental-features.md#xp-feature-blake-hashing) experimental feature.
|
||||
@@ -0,0 +1,2 @@
|
||||
# Process JSON schema documentation
|
||||
subdir('json')
|
||||
@@ -413,6 +413,10 @@
|
||||
supportsCross = false;
|
||||
};
|
||||
|
||||
"nix-json-schema-checks" = {
|
||||
supportsCross = false;
|
||||
};
|
||||
|
||||
"nix-perl-bindings" = {
|
||||
supportsCross = false;
|
||||
};
|
||||
|
||||
@@ -60,3 +60,4 @@ if get_option('unit-tests')
|
||||
subproject('libflake-tests')
|
||||
endif
|
||||
subproject('nix-functional-tests')
|
||||
subproject('json-schema-checks')
|
||||
@@ -438,6 +438,11 @@ in
|
||||
*/
|
||||
nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; };
|
||||
|
||||
/**
|
||||
JSON schema validation checks
|
||||
*/
|
||||
nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { };
|
||||
|
||||
nix-perl-bindings = callPackage ../src/perl/package.nix { };
|
||||
|
||||
/**
|
||||
|
||||
@@ -108,6 +108,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs
|
||||
++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs
|
||||
++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs
|
||||
++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs
|
||||
++ lib.optional (
|
||||
!buildCanExecuteHost
|
||||
# Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
|
||||
|
||||
@@ -62,6 +62,7 @@ let
|
||||
"nix-cmd"
|
||||
"nix-cli"
|
||||
"nix-functional-tests"
|
||||
"nix-json-schema-checks"
|
||||
]
|
||||
++ lib.optionals enableBindings [
|
||||
"nix-perl-bindings"
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../.version
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../src/libstore-tests/data/derivation
|
||||
@@ -0,0 +1 @@
|
||||
../../src/libutil-tests/data/memory-source-accessor
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../src/libutil-tests/data/hash
|
||||
@@ -0,0 +1,92 @@
|
||||
# Run with:
|
||||
# meson test --suite json-schema
|
||||
# Run with: (without shell / configure)
|
||||
# nix build .#nix-json-schema-checks
|
||||
|
||||
project(
|
||||
'nix-json-schema-checks',
|
||||
version : files('.version'),
|
||||
meson_version : '>= 1.1',
|
||||
license : 'LGPL-2.1-or-later',
|
||||
)
|
||||
|
||||
fs = import('fs')
|
||||
|
||||
# Note: The 'jsonschema' package provides the 'jv' command
|
||||
jv = find_program('jv', required : true)
|
||||
|
||||
# The schema directory is a committed symlink to the actual schema location
|
||||
schema_dir = meson.current_source_dir() / 'schema'
|
||||
|
||||
# Get all example files
|
||||
schemas = [
|
||||
{
|
||||
'stem' : 'file-system-object',
|
||||
'schema' : schema_dir / 'file-system-object-v1.yaml',
|
||||
'files' : [
|
||||
'simple.json',
|
||||
'complex.json',
|
||||
],
|
||||
},
|
||||
{
|
||||
'stem' : 'hash',
|
||||
'schema' : schema_dir / 'hash-v1.yaml',
|
||||
'files' : [
|
||||
'sha256-base64.json',
|
||||
'sha256-base16.json',
|
||||
'sha256-nix32.json',
|
||||
'blake3-base64.json',
|
||||
],
|
||||
},
|
||||
{
|
||||
'stem' : 'derivation',
|
||||
'schema' : schema_dir / 'derivation-v3.yaml',
|
||||
'files' : [
|
||||
'dyn-dep-derivation.json',
|
||||
'simple-derivation.json',
|
||||
],
|
||||
},
|
||||
# # Not sure how to make subschema work
|
||||
# {
|
||||
# 'stem': 'derivation',
|
||||
# 'schema': schema_dir / 'derivation-v3.yaml#output',
|
||||
# 'files' : [
|
||||
# 'output-caFixedFlat.json',
|
||||
# 'output-caFixedNAR.json',
|
||||
# 'output-caFixedText.json',
|
||||
# 'output-caFloating.json',
|
||||
# 'output-deferred.json',
|
||||
# 'output-impure.json',
|
||||
# 'output-inputAddressed.json',
|
||||
# ],
|
||||
# },
|
||||
]
|
||||
|
||||
# Validate each example against the schema
|
||||
foreach schema : schemas
|
||||
stem = schema['stem']
|
||||
schema_file = schema['schema']
|
||||
if '#' not in schema_file
|
||||
# Validate the schema itself against JSON Schema Draft 04
|
||||
test(
|
||||
stem + '-schema-valid',
|
||||
jv,
|
||||
args : [
|
||||
'http://json-schema.org/draft-04/schema',
|
||||
schema_file,
|
||||
],
|
||||
suite : 'json-schema',
|
||||
)
|
||||
endif
|
||||
foreach example : schema['files']
|
||||
test(
|
||||
stem + '-example-' + fs.stem(example),
|
||||
jv,
|
||||
args : [
|
||||
schema_file,
|
||||
files(stem / example),
|
||||
],
|
||||
suite : 'json-schema',
|
||||
)
|
||||
endforeach
|
||||
endforeach
|
||||
@@ -0,0 +1,52 @@
|
||||
# Run with: nix build .#nix-json-schema-checks
|
||||
{
|
||||
lib,
|
||||
mkMesonDerivation,
|
||||
|
||||
meson,
|
||||
ninja,
|
||||
jsonschema,
|
||||
|
||||
# Configuration Options
|
||||
|
||||
version,
|
||||
}:
|
||||
|
||||
mkMesonDerivation (finalAttrs: {
|
||||
pname = "nix-json-schema-checks";
|
||||
inherit version;
|
||||
|
||||
workDir = ./.;
|
||||
fileset = lib.fileset.unions [
|
||||
../../.version
|
||||
../../doc/manual/source/protocols/json/schema
|
||||
../../src/libutil-tests/data/memory-source-accessor
|
||||
../../src/libutil-tests/data/hash
|
||||
../../src/libstore-tests/data/derivation
|
||||
./.
|
||||
];
|
||||
|
||||
outputs = [ "out" ];
|
||||
|
||||
passthru.externalNativeBuildInputs = [
|
||||
jsonschema
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
meson
|
||||
ninja
|
||||
]
|
||||
++ finalAttrs.passthru.externalNativeBuildInputs;
|
||||
|
||||
doCheck = true;
|
||||
|
||||
mesonCheckFlags = [ "--print-errorlogs" ];
|
||||
|
||||
postInstall = ''
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta = {
|
||||
platforms = lib.platforms.all;
|
||||
};
|
||||
})
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../doc/manual/source/protocols/json/schema
|
||||
@@ -199,8 +199,8 @@ static void fetchTree(
|
||||
if (state.settings.pureEval && !input.isLocked()) {
|
||||
if (input.getNarHash())
|
||||
warn(
|
||||
"Input '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. "
|
||||
"This is deprecated since such inputs are verifiable but may not be reproducible.",
|
||||
"Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
|
||||
"This is not reproducible and will break after garbage collection or when shared.",
|
||||
input.to_string());
|
||||
else
|
||||
state
|
||||
|
||||
@@ -77,8 +77,8 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann:
|
||||
if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) {
|
||||
if (lockedRef.input.getNarHash())
|
||||
warn(
|
||||
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but does have a NAR hash. "
|
||||
"This is deprecated since such inputs are verifiable but may not be reproducible.",
|
||||
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
|
||||
"This is not reproducible and will break after garbage collection or when shared.",
|
||||
lockedRef.to_string());
|
||||
else
|
||||
throw Error(
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
using nlohmann::json;
|
||||
using namespace nlohmann;
|
||||
|
||||
class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest
|
||||
{
|
||||
@@ -51,44 +51,44 @@ using BothFixtures = ::testing::Types<DerivationAdvancedAttrsTest, CaDerivationA
|
||||
|
||||
TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures);
|
||||
|
||||
#define TEST_ATERM_JSON(STEM, NAME) \
|
||||
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \
|
||||
{ \
|
||||
this->readTest(NAME ".json", [&](const auto & encoded_) { \
|
||||
auto encoded = json::parse(encoded_); \
|
||||
/* Use DRV file instead of C++ literal as source of truth. */ \
|
||||
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
|
||||
auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
|
||||
Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \
|
||||
EXPECT_EQ(got, expected); \
|
||||
}); \
|
||||
} \
|
||||
\
|
||||
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \
|
||||
{ \
|
||||
this->writeTest( \
|
||||
NAME ".json", \
|
||||
[&]() -> json { \
|
||||
/* Use DRV file instead of C++ literal as source of truth. */ \
|
||||
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
|
||||
return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \
|
||||
}, \
|
||||
[](const auto & file) { return json::parse(readFile(file)); }, \
|
||||
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
|
||||
} \
|
||||
\
|
||||
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \
|
||||
{ \
|
||||
this->readTest(NAME ".drv", [&](auto encoded) { \
|
||||
/* Use JSON file instead of C++ literal as source of truth. */ \
|
||||
auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \
|
||||
auto expected = Derivation::fromJSON(json, this->mockXpSettings); \
|
||||
auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \
|
||||
EXPECT_EQ(got.toJSON(), expected.toJSON()); \
|
||||
EXPECT_EQ(got, expected); \
|
||||
}); \
|
||||
} \
|
||||
\
|
||||
#define TEST_ATERM_JSON(STEM, NAME) \
|
||||
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \
|
||||
{ \
|
||||
this->readTest(NAME ".json", [&](const auto & encoded_) { \
|
||||
auto encoded = json::parse(encoded_); \
|
||||
/* Use DRV file instead of C++ literal as source of truth. */ \
|
||||
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
|
||||
auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
|
||||
Derivation got = adl_serializer<Derivation>::from_json(encoded, this->mockXpSettings); \
|
||||
EXPECT_EQ(got, expected); \
|
||||
}); \
|
||||
} \
|
||||
\
|
||||
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \
|
||||
{ \
|
||||
this->writeTest( \
|
||||
NAME ".json", \
|
||||
[&]() -> json { \
|
||||
/* Use DRV file instead of C++ literal as source of truth. */ \
|
||||
auto aterm = readFile(this->goldenMaster(NAME ".drv")); \
|
||||
return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \
|
||||
}, \
|
||||
[](const auto & file) { return json::parse(readFile(file)); }, \
|
||||
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
|
||||
} \
|
||||
\
|
||||
TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \
|
||||
{ \
|
||||
this->readTest(NAME ".drv", [&](auto encoded) { \
|
||||
/* Use JSON file instead of C++ literal as source of truth. */ \
|
||||
auto j = json::parse(readFile(this->goldenMaster(NAME ".json"))); \
|
||||
auto expected = adl_serializer<Derivation>::from_json(j, this->mockXpSettings); \
|
||||
auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \
|
||||
EXPECT_EQ(static_cast<json>(got), static_cast<json>(expected)); \
|
||||
EXPECT_EQ(got, expected); \
|
||||
}); \
|
||||
} \
|
||||
\
|
||||
/* No corresponding write test, because we need to read the drv to write the json file */
|
||||
|
||||
TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults");
|
||||
|
||||
@@ -66,23 +66,17 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps)
|
||||
FormatError);
|
||||
}
|
||||
|
||||
#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \
|
||||
TEST_P(FIXTURE, from_json) \
|
||||
{ \
|
||||
const auto & [name, expected] = GetParam(); \
|
||||
/* Don't use readJsonTest because we want to check experimental \
|
||||
features. */ \
|
||||
readTest(Path{"output-"} + name + ".json", [&](const auto & encoded_) { \
|
||||
json j = json::parse(encoded_); \
|
||||
DerivationOutput got = DerivationOutput::fromJSON(j, mockXpSettings); \
|
||||
ASSERT_EQ(got, expected); \
|
||||
}); \
|
||||
} \
|
||||
\
|
||||
TEST_P(FIXTURE, to_json) \
|
||||
{ \
|
||||
const auto & [name, value] = GetParam(); \
|
||||
writeJsonTest("output-" + name, value); \
|
||||
#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \
|
||||
TEST_P(FIXTURE, from_json) \
|
||||
{ \
|
||||
const auto & [name, expected] = GetParam(); \
|
||||
readJsonTest(Path{"output-"} + name, expected, mockXpSettings); \
|
||||
} \
|
||||
\
|
||||
TEST_P(FIXTURE, to_json) \
|
||||
{ \
|
||||
const auto & [name, value] = GetParam(); \
|
||||
writeJsonTest("output-" + name, value); \
|
||||
}
|
||||
|
||||
struct DerivationOutputJsonTest : DerivationTest,
|
||||
@@ -193,13 +187,7 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
TEST_P(FIXTURE, from_json) \
|
||||
{ \
|
||||
const auto & drv = GetParam(); \
|
||||
/* Don't use readJsonTest because we want to check experimental \
|
||||
features. */ \
|
||||
readTest(drv.name + ".json", [&](const auto & encoded_) { \
|
||||
auto encoded = json::parse(encoded_); \
|
||||
Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \
|
||||
ASSERT_EQ(got, drv); \
|
||||
}); \
|
||||
readJsonTest(drv.name, drv, mockXpSettings); \
|
||||
} \
|
||||
\
|
||||
TEST_P(FIXTURE, to_json) \
|
||||
@@ -213,7 +201,8 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
const auto & drv = GetParam(); \
|
||||
readTest(drv.name + ".drv", [&](auto encoded) { \
|
||||
auto got = parseDerivation(*store, std::move(encoded), drv.name, mockXpSettings); \
|
||||
ASSERT_EQ(got.toJSON(), drv.toJSON()); \
|
||||
using nlohmann::json; \
|
||||
ASSERT_EQ(static_cast<json>(got), static_cast<json>(drv)); \
|
||||
ASSERT_EQ(got, drv); \
|
||||
}); \
|
||||
} \
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <rapidcheck/gtest.h>
|
||||
|
||||
#include "nix/util/tests/characterization.hh"
|
||||
#include "nix/util/tests/json-characterization.hh"
|
||||
#include "nix/store/tests/derived-path.hh"
|
||||
#include "nix/store/tests/libstore.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class DerivedPathTest : public CharacterizationTest, public LibStoreTest
|
||||
class DerivedPathTest : public virtual CharacterizationTest, public LibStoreTest
|
||||
{
|
||||
std::filesystem::path unitTestData = getUnitTestData() / "derived-path";
|
||||
|
||||
@@ -123,25 +123,51 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o))
|
||||
|
||||
using nlohmann::json;
|
||||
|
||||
#define TEST_JSON(TYPE, NAME, VAL) \
|
||||
static const TYPE NAME = VAL; \
|
||||
\
|
||||
TEST_F(DerivedPathTest, NAME##_from_json) \
|
||||
{ \
|
||||
readTest(#NAME ".json", [&](const auto & encoded_) { \
|
||||
auto encoded = json::parse(encoded_); \
|
||||
TYPE got = static_cast<TYPE>(encoded); \
|
||||
ASSERT_EQ(got, NAME); \
|
||||
}); \
|
||||
} \
|
||||
\
|
||||
TEST_F(DerivedPathTest, NAME##_to_json) \
|
||||
{ \
|
||||
writeTest( \
|
||||
#NAME ".json", \
|
||||
[&]() -> json { return static_cast<json>(NAME); }, \
|
||||
[](const auto & file) { return json::parse(readFile(file)); }, \
|
||||
[](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \
|
||||
struct SingleDerivedPathJsonTest : DerivedPathTest,
|
||||
JsonCharacterizationTest<SingleDerivedPath>,
|
||||
::testing::WithParamInterface<SingleDerivedPath>
|
||||
{};
|
||||
|
||||
struct DerivedPathJsonTest : DerivedPathTest,
|
||||
JsonCharacterizationTest<DerivedPath>,
|
||||
::testing::WithParamInterface<DerivedPath>
|
||||
{};
|
||||
|
||||
#define TEST_JSON(TYPE, NAME, VAL) \
|
||||
static const TYPE NAME = VAL; \
|
||||
\
|
||||
TEST_F(TYPE##JsonTest, NAME##_from_json) \
|
||||
{ \
|
||||
readJsonTest(#NAME, NAME); \
|
||||
} \
|
||||
\
|
||||
TEST_F(TYPE##JsonTest, NAME##_to_json) \
|
||||
{ \
|
||||
writeJsonTest(#NAME, NAME); \
|
||||
}
|
||||
|
||||
#define TEST_JSON_XP_DYN(TYPE, NAME, VAL) \
|
||||
static const TYPE NAME = VAL; \
|
||||
\
|
||||
TEST_F(TYPE##JsonTest, NAME##_from_json_throws_without_xp) \
|
||||
{ \
|
||||
std::optional<json> ret; \
|
||||
readTest(#NAME ".json", [&](const auto & encoded_) { ret = json::parse(encoded_); }); \
|
||||
if (ret) { \
|
||||
EXPECT_THROW(nlohmann::adl_serializer<TYPE>::from_json(*ret), MissingExperimentalFeature); \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
TEST_F(TYPE##JsonTest, NAME##_from_json) \
|
||||
{ \
|
||||
ExperimentalFeatureSettings xpSettings; \
|
||||
xpSettings.set("experimental-features", "dynamic-derivations"); \
|
||||
readJsonTest(#NAME, NAME, xpSettings); \
|
||||
} \
|
||||
\
|
||||
TEST_F(TYPE##JsonTest, NAME##_to_json) \
|
||||
{ \
|
||||
writeJsonTest(#NAME, NAME); \
|
||||
}
|
||||
|
||||
TEST_JSON(
|
||||
@@ -156,7 +182,7 @@ TEST_JSON(
|
||||
.output = "bar",
|
||||
}));
|
||||
|
||||
TEST_JSON(
|
||||
TEST_JSON_XP_DYN(
|
||||
SingleDerivedPath,
|
||||
single_built_built,
|
||||
(SingleDerivedPath::Built{
|
||||
@@ -179,7 +205,7 @@ TEST_JSON(
|
||||
.outputs = OutputsSpec::Names{"bar", "baz"},
|
||||
}));
|
||||
|
||||
TEST_JSON(
|
||||
TEST_JSON_XP_DYN(
|
||||
DerivedPath,
|
||||
multi_built_built,
|
||||
(DerivedPath::Built{
|
||||
@@ -191,7 +217,7 @@ TEST_JSON(
|
||||
.outputs = OutputsSpec::Names{"baz", "quux"},
|
||||
}));
|
||||
|
||||
TEST_JSON(
|
||||
TEST_JSON_XP_DYN(
|
||||
DerivedPath,
|
||||
multi_built_built_wildcard,
|
||||
(DerivedPath::Built{
|
||||
|
||||
+36
-46
@@ -96,67 +96,57 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createProviderForProfile(const std::string & profile);
|
||||
|
||||
private:
|
||||
Aws::Crt::ApiHandle apiHandle;
|
||||
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
|
||||
credentialProviderCache;
|
||||
};
|
||||
|
||||
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
|
||||
AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
|
||||
{
|
||||
debug(
|
||||
"[pid=%d] creating new AWS credential provider for profile '%s'",
|
||||
getpid(),
|
||||
profile.empty() ? "(default)" : profile.c_str());
|
||||
|
||||
if (profile.empty()) {
|
||||
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
|
||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
|
||||
}
|
||||
|
||||
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
|
||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
// This is safe because the underlying C library will copy this string
|
||||
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
|
||||
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
|
||||
}
|
||||
|
||||
AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
|
||||
{
|
||||
// Get or create credential provider with caching
|
||||
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider;
|
||||
|
||||
// Use try_emplace_and_cvisit for atomic get-or-create
|
||||
// This prevents race conditions where multiple threads create providers
|
||||
credentialProviderCache.try_emplace_and_cvisit(
|
||||
profile,
|
||||
nullptr, // Placeholder - will be replaced in f1 before any thread can see it
|
||||
[&](auto & kv) {
|
||||
// f1: Called atomically during insertion with non-const reference
|
||||
// Other threads are blocked until we finish, so nullptr is never visible
|
||||
debug(
|
||||
"[pid=%d] creating new AWS credential provider for profile '%s'",
|
||||
getpid(),
|
||||
profile.empty() ? "(default)" : profile.c_str());
|
||||
nullptr,
|
||||
[&](auto & kv) { provider = kv.second = createProviderForProfile(profile); },
|
||||
[&](const auto & kv) { provider = kv.second; });
|
||||
|
||||
try {
|
||||
if (profile.empty()) {
|
||||
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
|
||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
|
||||
} else {
|
||||
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
|
||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||
// This is safe because the underlying C library will copy this string
|
||||
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
|
||||
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
||||
kv.second = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
|
||||
}
|
||||
|
||||
if (!kv.second) {
|
||||
throw AwsAuthError(
|
||||
"Failed to create AWS credentials provider for %s",
|
||||
profile.empty() ? "default profile" : fmt("profile '%s'", profile));
|
||||
}
|
||||
|
||||
provider = kv.second;
|
||||
} catch (Error & e) {
|
||||
// Exception during creation - remove the entry to allow retry
|
||||
credentialProviderCache.erase(profile);
|
||||
e.addTrace({}, "for AWS profile: %s", profile.empty() ? "(default)" : profile);
|
||||
throw;
|
||||
} catch (...) {
|
||||
// Non-Error exception - still need to clean up
|
||||
credentialProviderCache.erase(profile);
|
||||
throw;
|
||||
}
|
||||
},
|
||||
[&](const auto & kv) {
|
||||
// f2: Called if key already exists (const reference)
|
||||
provider = kv.second;
|
||||
if (!provider) {
|
||||
credentialProviderCache.erase_if(profile, [](const auto & kv) {
|
||||
[[maybe_unused]] auto [_, provider] = kv;
|
||||
return !provider;
|
||||
});
|
||||
|
||||
throw AwsAuthError(
|
||||
"Failed to create AWS credentials provider for %s",
|
||||
profile.empty() ? "default profile" : fmt("profile '%s'", profile));
|
||||
}
|
||||
|
||||
return getCredentialsFromProvider(provider);
|
||||
}
|
||||
|
||||
|
||||
@@ -112,7 +112,21 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
if (failed)
|
||||
continue;
|
||||
|
||||
co_return realisationFetched(std::move(waitees), outputInfo, sub);
|
||||
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
|
||||
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
|
||||
}
|
||||
|
||||
worker.store.registerDrvOutput({*outputInfo, id});
|
||||
|
||||
trace("finished");
|
||||
co_return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
@@ -130,26 +144,6 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters);
|
||||
}
|
||||
|
||||
Goal::Co DrvOutputSubstitutionGoal::realisationFetched(
|
||||
Goals waitees, std::shared_ptr<const UnkeyedRealisation> outputInfo, nix::ref<nix::Store> sub)
|
||||
{
|
||||
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
|
||||
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
|
||||
}
|
||||
|
||||
worker.store.registerDrvOutput({*outputInfo, id});
|
||||
|
||||
trace("finished");
|
||||
co_return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
std::string DrvOutputSubstitutionGoal::key()
|
||||
{
|
||||
return "a$" + std::string(id.to_string());
|
||||
|
||||
+24
-47
@@ -1261,9 +1261,15 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
|
||||
|
||||
const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure");
|
||||
|
||||
nlohmann::json DerivationOutput::toJSON() const
|
||||
} // namespace nix
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
using namespace nix;
|
||||
|
||||
void adl_serializer<DerivationOutput>::to_json(json & res, const DerivationOutput & o)
|
||||
{
|
||||
nlohmann::json res = nlohmann::json::object();
|
||||
res = nlohmann::json::object();
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; },
|
||||
@@ -1289,12 +1295,11 @@ nlohmann::json DerivationOutput::toJSON() const
|
||||
res["impure"] = true;
|
||||
},
|
||||
},
|
||||
raw);
|
||||
return res;
|
||||
o.raw);
|
||||
}
|
||||
|
||||
DerivationOutput
|
||||
DerivationOutput::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings)
|
||||
adl_serializer<DerivationOutput>::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
std::set<std::string_view> keys;
|
||||
auto & json = getObject(_json);
|
||||
@@ -1362,18 +1367,18 @@ DerivationOutput::fromJSON(const nlohmann::json & _json, const ExperimentalFeatu
|
||||
}
|
||||
}
|
||||
|
||||
nlohmann::json Derivation::toJSON() const
|
||||
void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
|
||||
{
|
||||
nlohmann::json res = nlohmann::json::object();
|
||||
res = nlohmann::json::object();
|
||||
|
||||
res["name"] = name;
|
||||
res["name"] = d.name;
|
||||
|
||||
res["version"] = 3;
|
||||
|
||||
{
|
||||
nlohmann::json & outputsObj = res["outputs"];
|
||||
outputsObj = nlohmann::json::object();
|
||||
for (auto & [outputName, output] : outputs) {
|
||||
for (auto & [outputName, output] : d.outputs) {
|
||||
outputsObj[outputName] = output;
|
||||
}
|
||||
}
|
||||
@@ -1381,7 +1386,7 @@ nlohmann::json Derivation::toJSON() const
|
||||
{
|
||||
auto & inputsList = res["inputSrcs"];
|
||||
inputsList = nlohmann::json ::array();
|
||||
for (auto & input : inputSrcs)
|
||||
for (auto & input : d.inputSrcs)
|
||||
inputsList.emplace_back(input);
|
||||
}
|
||||
|
||||
@@ -1401,24 +1406,22 @@ nlohmann::json Derivation::toJSON() const
|
||||
{
|
||||
auto & inputDrvsObj = res["inputDrvs"];
|
||||
inputDrvsObj = nlohmann::json::object();
|
||||
for (auto & [inputDrv, inputNode] : inputDrvs.map) {
|
||||
for (auto & [inputDrv, inputNode] : d.inputDrvs.map) {
|
||||
inputDrvsObj[inputDrv.to_string()] = doInput(inputNode);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res["system"] = platform;
|
||||
res["builder"] = builder;
|
||||
res["args"] = args;
|
||||
res["env"] = env;
|
||||
res["system"] = d.platform;
|
||||
res["builder"] = d.builder;
|
||||
res["args"] = d.args;
|
||||
res["env"] = d.env;
|
||||
|
||||
if (structuredAttrs)
|
||||
res["structuredAttrs"] = structuredAttrs->structuredAttrs;
|
||||
|
||||
return res;
|
||||
if (d.structuredAttrs)
|
||||
res["structuredAttrs"] = d.structuredAttrs->structuredAttrs;
|
||||
}
|
||||
|
||||
Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings)
|
||||
Derivation adl_serializer<Derivation>::from_json(const json & _json, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
using nlohmann::detail::value_t;
|
||||
|
||||
@@ -1434,7 +1437,7 @@ Derivation Derivation::fromJSON(const nlohmann::json & _json, const Experimental
|
||||
try {
|
||||
auto outputs = getObject(valueAt(json, "outputs"));
|
||||
for (auto & [outputName, output] : outputs) {
|
||||
res.outputs.insert_or_assign(outputName, DerivationOutput::fromJSON(output, xpSettings));
|
||||
res.outputs.insert_or_assign(outputName, adl_serializer<DerivationOutput>::from_json(output, xpSettings));
|
||||
}
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while reading key 'outputs'");
|
||||
@@ -1490,30 +1493,4 @@ Derivation Derivation::fromJSON(const nlohmann::json & _json, const Experimental
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
using namespace nix;
|
||||
|
||||
DerivationOutput adl_serializer<DerivationOutput>::from_json(const json & json)
|
||||
{
|
||||
return DerivationOutput::fromJSON(json);
|
||||
}
|
||||
|
||||
void adl_serializer<DerivationOutput>::to_json(json & json, const DerivationOutput & c)
|
||||
{
|
||||
json = c.toJSON();
|
||||
}
|
||||
|
||||
Derivation adl_serializer<Derivation>::from_json(const json & json)
|
||||
{
|
||||
return Derivation::fromJSON(json);
|
||||
}
|
||||
|
||||
void adl_serializer<Derivation>::to_json(json & json, const Derivation & c)
|
||||
{
|
||||
json = c.toJSON();
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
@@ -252,20 +252,26 @@ void adl_serializer<DerivedPath::Built>::to_json(json & json, const DerivedPath:
|
||||
};
|
||||
}
|
||||
|
||||
SingleDerivedPath::Built adl_serializer<SingleDerivedPath::Built>::from_json(const json & json0)
|
||||
SingleDerivedPath::Built
|
||||
adl_serializer<SingleDerivedPath::Built>::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
auto & json = getObject(json0);
|
||||
auto drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath")));
|
||||
drvRequireExperiment(*drvPath, xpSettings);
|
||||
return {
|
||||
.drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath"))),
|
||||
.drvPath = std::move(drvPath),
|
||||
.output = getString(valueAt(json, "output")),
|
||||
};
|
||||
}
|
||||
|
||||
DerivedPath::Built adl_serializer<DerivedPath::Built>::from_json(const json & json0)
|
||||
DerivedPath::Built
|
||||
adl_serializer<DerivedPath::Built>::from_json(const json & json0, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
auto & json = getObject(json0);
|
||||
auto drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath")));
|
||||
drvRequireExperiment(*drvPath, xpSettings);
|
||||
return {
|
||||
.drvPath = make_ref<SingleDerivedPath>(static_cast<SingleDerivedPath>(valueAt(json, "drvPath"))),
|
||||
.drvPath = std::move(drvPath),
|
||||
.outputs = adl_serializer<OutputsSpec>::from_json(valueAt(json, "outputs")),
|
||||
};
|
||||
}
|
||||
@@ -280,20 +286,21 @@ void adl_serializer<DerivedPath>::to_json(json & json, const DerivedPath & sdp)
|
||||
std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw());
|
||||
}
|
||||
|
||||
SingleDerivedPath adl_serializer<SingleDerivedPath>::from_json(const json & json)
|
||||
SingleDerivedPath
|
||||
adl_serializer<SingleDerivedPath>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
if (json.is_string())
|
||||
return static_cast<SingleDerivedPath::Opaque>(json);
|
||||
else
|
||||
return static_cast<SingleDerivedPath::Built>(json);
|
||||
return adl_serializer<SingleDerivedPath::Built>::from_json(json, xpSettings);
|
||||
}
|
||||
|
||||
DerivedPath adl_serializer<DerivedPath>::from_json(const json & json)
|
||||
DerivedPath adl_serializer<DerivedPath>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
if (json.is_string())
|
||||
return static_cast<DerivedPath::Opaque>(json);
|
||||
else
|
||||
return static_cast<DerivedPath::Built>(json);
|
||||
return adl_serializer<DerivedPath::Built>::from_json(json, xpSettings);
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
@@ -32,8 +32,6 @@ public:
|
||||
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
|
||||
|
||||
Co init();
|
||||
Co
|
||||
realisationFetched(Goals waitees, std::shared_ptr<const UnkeyedRealisation> outputInfo, nix::ref<nix::Store> sub);
|
||||
|
||||
void timedOut(Error && ex) override
|
||||
{
|
||||
|
||||
@@ -134,13 +134,6 @@ struct DerivationOutput
|
||||
*/
|
||||
std::optional<StorePath>
|
||||
path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const;
|
||||
|
||||
nlohmann::json toJSON() const;
|
||||
/**
|
||||
* @param xpSettings Stop-gap to avoid globals during unit tests.
|
||||
*/
|
||||
static DerivationOutput
|
||||
fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
};
|
||||
|
||||
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
|
||||
@@ -390,10 +383,6 @@ struct Derivation : BasicDerivation
|
||||
{
|
||||
}
|
||||
|
||||
nlohmann::json toJSON() const;
|
||||
static Derivation
|
||||
fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
|
||||
bool operator==(const Derivation &) const = default;
|
||||
// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
|
||||
// auto operator <=> (const Derivation &) const = default;
|
||||
@@ -537,5 +526,5 @@ std::string hashPlaceholder(const OutputNameView outputName);
|
||||
|
||||
} // namespace nix
|
||||
|
||||
JSON_IMPL(nix::DerivationOutput)
|
||||
JSON_IMPL(nix::Derivation)
|
||||
JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)
|
||||
JSON_IMPL_WITH_XP_FEATURES(nix::Derivation)
|
||||
@@ -299,7 +299,7 @@ void drvRequireExperiment(
|
||||
} // namespace nix
|
||||
|
||||
JSON_IMPL(nix::SingleDerivedPath::Opaque)
|
||||
JSON_IMPL(nix::SingleDerivedPath::Built)
|
||||
JSON_IMPL(nix::SingleDerivedPath)
|
||||
JSON_IMPL(nix::DerivedPath::Built)
|
||||
JSON_IMPL(nix::DerivedPath)
|
||||
JSON_IMPL_WITH_XP_FEATURES(nix::SingleDerivedPath::Built)
|
||||
JSON_IMPL_WITH_XP_FEATURES(nix::SingleDerivedPath)
|
||||
JSON_IMPL_WITH_XP_FEATURES(nix::DerivedPath::Built)
|
||||
JSON_IMPL_WITH_XP_FEATURES(nix::DerivedPath)
|
||||
@@ -113,7 +113,7 @@ boost = dependency(
|
||||
# put in `deps_other`.
|
||||
deps_other += boost
|
||||
|
||||
curl = dependency('libcurl', 'curl')
|
||||
curl = dependency('libcurl', 'curl', version : '>= 7.75.0')
|
||||
deps_private += curl
|
||||
|
||||
# seccomp only makes sense on Linux
|
||||
@@ -142,27 +142,16 @@ deps_public += nlohmann_json
|
||||
sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19')
|
||||
deps_private += sqlite
|
||||
|
||||
# Curl-based S3 store support
|
||||
# Check if curl supports AWS SigV4 (requires >= 7.75.0)
|
||||
curl_supports_aws_sigv4 = curl.version().version_compare('>= 7.75.0')
|
||||
# AWS CRT C++ for lightweight credential management
|
||||
aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : false)
|
||||
s3_aws_auth = get_option('s3-aws-auth')
|
||||
aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : s3_aws_auth)
|
||||
|
||||
curl_s3_store_opt = get_option('curl-s3-store').require(
|
||||
curl_supports_aws_sigv4,
|
||||
error_message : 'curl-based S3 support requires curl >= 7.75.0',
|
||||
).require(
|
||||
aws_crt_cpp.found(),
|
||||
error_message : 'curl-based S3 support requires aws-crt-cpp',
|
||||
)
|
||||
|
||||
if curl_s3_store_opt.enabled()
|
||||
if s3_aws_auth.enabled()
|
||||
deps_other += aws_crt_cpp
|
||||
aws_c_common = cxx.find_library('aws-c-common', required : true)
|
||||
deps_other += aws_c_common
|
||||
endif
|
||||
|
||||
configdata_pub.set('NIX_WITH_AWS_AUTH', curl_s3_store_opt.enabled().to_int())
|
||||
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())
|
||||
|
||||
subdir('nix-meson-build-support/generate-header')
|
||||
|
||||
@@ -346,7 +335,7 @@ sources = files(
|
||||
)
|
||||
|
||||
# AWS credentials code requires AWS CRT, so only compile when enabled
|
||||
if curl_s3_store_opt.enabled()
|
||||
if s3_aws_auth.enabled()
|
||||
sources += files('aws-creds.cc')
|
||||
endif
|
||||
|
||||
|
||||
@@ -35,8 +35,7 @@ option(
|
||||
)
|
||||
|
||||
option(
|
||||
'curl-s3-store',
|
||||
's3-aws-auth',
|
||||
type : 'feature',
|
||||
value : 'disabled',
|
||||
description : 'Enable curl-based S3 binary cache store support (requires aws-crt-cpp and curl >= 7.75.0)',
|
||||
description : 'build support for AWS authentication with S3',
|
||||
)
|
||||
@@ -75,7 +75,7 @@ mkMesonLibrary (finalAttrs: {
|
||||
mesonFlags = [
|
||||
(lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
|
||||
(lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
|
||||
(lib.mesonEnable "curl-s3-store" withAWS)
|
||||
(lib.mesonEnable "s3-aws-auth" withAWS)
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isLinux [
|
||||
(lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
|
||||
|
||||
@@ -24,12 +24,12 @@ struct JsonCharacterizationTest : virtual CharacterizationTest
|
||||
* @param test hook that takes the contents of the file and does the
|
||||
* actual work
|
||||
*/
|
||||
void readJsonTest(PathView testStem, const T & expected)
|
||||
void readJsonTest(PathView testStem, const T & expected, auto... args)
|
||||
{
|
||||
using namespace nlohmann;
|
||||
readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) {
|
||||
auto encoded = json::parse(encodedRaw);
|
||||
T decoded = adl_serializer<T>::from_json(encoded);
|
||||
T decoded = adl_serializer<T>::from_json(encoded, args...);
|
||||
ASSERT_EQ(decoded, expected);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#include "nix/util/alignment.hh"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
TEST(alignUp, value)
|
||||
{
|
||||
for (uint64_t i = 1; i <= 8; ++i)
|
||||
EXPECT_EQ(alignUp(i, 8), 8);
|
||||
}
|
||||
|
||||
TEST(alignUp, notAPowerOf2)
|
||||
{
|
||||
ASSERT_DEATH({ alignUp(1u, 42); }, "alignment must be a power of 2");
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"algorithm": "blake3",
|
||||
"format": "base64",
|
||||
"hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA="
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"algorithm": "sha256",
|
||||
"format": "base16",
|
||||
"hash": "f0e4c2f76c58916ec258f246851bea091d14d4247a2fc3e18694461b1816e13b"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"algorithm": "sha256",
|
||||
"format": "base64",
|
||||
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"algorithm": "sha256",
|
||||
"format": "nix32",
|
||||
"hash": "0fz12qc1nillhvhw6bvs4ka18789x8dqaipjb316x4aqdkvw5r7h"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"algorithm": "sha256",
|
||||
"format": "base64",
|
||||
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"contents": {
|
||||
"bar": {
|
||||
"contents": {
|
||||
"baz": {
|
||||
"contents": "Z29vZCBkYXksCg==",
|
||||
"executable": true,
|
||||
"type": "regular"
|
||||
},
|
||||
"quux": {
|
||||
"target": "/over/there",
|
||||
"type": "symlink"
|
||||
}
|
||||
},
|
||||
"type": "directory"
|
||||
},
|
||||
"foo": {
|
||||
"contents": "aGVsbG8K",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
}
|
||||
},
|
||||
"type": "directory"
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"contents": "YXNkZg==",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
}
|
||||
@@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write)
|
||||
});
|
||||
}
|
||||
|
||||
namespace memory_source_accessor {
|
||||
|
||||
extern ref<MemorySourceAccessor> exampleComplex();
|
||||
|
||||
}
|
||||
|
||||
TEST_F(GitTest, both_roundrip)
|
||||
{
|
||||
using File = MemorySourceAccessor::File;
|
||||
|
||||
auto files = make_ref<MemorySourceAccessor>();
|
||||
files->root = File::Directory{
|
||||
.contents{
|
||||
{
|
||||
"foo",
|
||||
File::Regular{
|
||||
.contents = "hello\n\0\n\tworld!",
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
File::Directory{
|
||||
.contents =
|
||||
{
|
||||
{
|
||||
"baz",
|
||||
File::Regular{
|
||||
.executable = true,
|
||||
.contents = "good day,\n\0\n\tworld!",
|
||||
},
|
||||
},
|
||||
{
|
||||
"quux",
|
||||
File::Symlink{
|
||||
.target = "/over/there",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
auto files = memory_source_accessor::exampleComplex();
|
||||
|
||||
for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) {
|
||||
std::map<Hash, std::string> cas;
|
||||
|
||||
+102
-9
@@ -4,30 +4,30 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "nix/util/hash.hh"
|
||||
#include "nix/util/tests/characterization.hh"
|
||||
#include "nix/util/tests/json-characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class HashTest : public CharacterizationTest
|
||||
class HashTest : public virtual CharacterizationTest
|
||||
{
|
||||
std::filesystem::path unitTestData = getUnitTestData() / "hash";
|
||||
|
||||
public:
|
||||
|
||||
/**
|
||||
* We set these in tests rather than the regular globals so we don't have
|
||||
* to worry about race conditions if the tests run concurrently.
|
||||
*/
|
||||
ExperimentalFeatureSettings mockXpSettings;
|
||||
|
||||
std::filesystem::path goldenMaster(std::string_view testStem) const override
|
||||
{
|
||||
return unitTestData / testStem;
|
||||
}
|
||||
};
|
||||
|
||||
class BLAKE3HashTest : public HashTest
|
||||
struct BLAKE3HashTest : virtual HashTest
|
||||
{
|
||||
/**
|
||||
* We set these in tests rather than the regular globals so we don't have
|
||||
* to worry about race conditions if the tests run concurrently.
|
||||
*/
|
||||
ExperimentalFeatureSettings mockXpSettings;
|
||||
|
||||
void SetUp() override
|
||||
{
|
||||
mockXpSettings.set("experimental-features", "blake3-hashes");
|
||||
@@ -203,4 +203,97 @@ TEST(hashFormat, testParseHashFormatOptException)
|
||||
{
|
||||
ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt);
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* JSON
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
using nlohmann::json;
|
||||
|
||||
struct HashJsonTest : virtual HashTest,
|
||||
JsonCharacterizationTest<Hash>,
|
||||
::testing::WithParamInterface<std::pair<std::string_view, Hash>>
|
||||
{};
|
||||
|
||||
struct HashJsonParseOnlyTest : virtual HashTest,
|
||||
JsonCharacterizationTest<Hash>,
|
||||
::testing::WithParamInterface<std::pair<std::string_view, Hash>>
|
||||
{};
|
||||
|
||||
struct BLAKE3HashJsonTest : virtual HashTest,
|
||||
BLAKE3HashTest,
|
||||
JsonCharacterizationTest<Hash>,
|
||||
::testing::WithParamInterface<std::pair<std::string_view, Hash>>
|
||||
{};
|
||||
|
||||
TEST_P(HashJsonTest, from_json)
|
||||
{
|
||||
auto & [name, expected] = GetParam();
|
||||
readJsonTest(name, expected);
|
||||
}
|
||||
|
||||
TEST_P(HashJsonTest, to_json)
|
||||
{
|
||||
auto & [name, value] = GetParam();
|
||||
writeJsonTest(name, value);
|
||||
}
|
||||
|
||||
TEST_P(HashJsonParseOnlyTest, from_json)
|
||||
{
|
||||
auto & [name, expected] = GetParam();
|
||||
readJsonTest(name, expected);
|
||||
}
|
||||
|
||||
TEST_P(BLAKE3HashJsonTest, from_json)
|
||||
{
|
||||
auto & [name, expected] = GetParam();
|
||||
readJsonTest(name, expected, mockXpSettings);
|
||||
}
|
||||
|
||||
TEST_P(BLAKE3HashJsonTest, to_json)
|
||||
{
|
||||
auto & [name, expected] = GetParam();
|
||||
writeJsonTest(name, expected);
|
||||
}
|
||||
|
||||
// Round-trip tests (from_json + to_json) for base64 format only
|
||||
// (to_json always outputs base64)
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
HashJSON,
|
||||
HashJsonTest,
|
||||
::testing::Values(
|
||||
std::pair{
|
||||
"simple",
|
||||
hashString(HashAlgorithm::SHA256, "asdf"),
|
||||
},
|
||||
std::pair{
|
||||
"sha256-base64",
|
||||
hashString(HashAlgorithm::SHA256, "asdf"),
|
||||
}));
|
||||
|
||||
// Parse-only tests for non-base64 formats
|
||||
// These verify C++ can deserialize other formats correctly
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
HashJSONParseOnly,
|
||||
HashJsonParseOnlyTest,
|
||||
::testing::Values(
|
||||
std::pair{
|
||||
"sha256-base16",
|
||||
hashString(HashAlgorithm::SHA256, "asdf"),
|
||||
},
|
||||
std::pair{
|
||||
"sha256-nix32",
|
||||
hashString(HashAlgorithm::SHA256, "asdf"),
|
||||
}));
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(BLAKE3HashJSONParseOnly, BLAKE3HashJsonTest, ([] {
|
||||
ExperimentalFeatureSettings mockXpSettings;
|
||||
mockXpSettings.set("experimental-features", "blake3-hashes");
|
||||
return ::testing::Values(
|
||||
std::pair{
|
||||
"blake3-base64",
|
||||
hashString(HashAlgorithm::BLAKE3, "asdf", mockXpSettings),
|
||||
});
|
||||
}()));
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,116 @@
|
||||
#include <string_view>
|
||||
|
||||
#include "nix/util/bytes.hh"
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
#include "nix/util/tests/json-characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
namespace memory_source_accessor {
|
||||
|
||||
using File = MemorySourceAccessor::File;
|
||||
|
||||
ref<MemorySourceAccessor> exampleSimple()
|
||||
{
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = File{File::Regular{
|
||||
.executable = false,
|
||||
.contents = to_owned(as_bytes("asdf")),
|
||||
}};
|
||||
return sc;
|
||||
}
|
||||
|
||||
ref<MemorySourceAccessor> exampleComplex()
|
||||
{
|
||||
auto files = make_ref<MemorySourceAccessor>();
|
||||
files->root = File::Directory{
|
||||
.contents{
|
||||
{
|
||||
"foo",
|
||||
File::Regular{
|
||||
.contents = to_owned(as_bytes("hello\n\0\n\tworld!")),
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
File::Directory{
|
||||
.contents =
|
||||
{
|
||||
{
|
||||
"baz",
|
||||
File::Regular{
|
||||
.executable = true,
|
||||
.contents = to_owned(as_bytes("good day,\n\0\n\tworld!")),
|
||||
},
|
||||
},
|
||||
{
|
||||
"quux",
|
||||
File::Symlink{
|
||||
.target = "/over/there",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
return files;
|
||||
}
|
||||
|
||||
} // namespace memory_source_accessor
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* JSON
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
class MemorySourceAccessorTest : public virtual CharacterizationTest
|
||||
{
|
||||
std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor";
|
||||
|
||||
public:
|
||||
|
||||
std::filesystem::path goldenMaster(std::string_view testStem) const override
|
||||
{
|
||||
return unitTestData / testStem;
|
||||
}
|
||||
};
|
||||
|
||||
using nlohmann::json;
|
||||
|
||||
struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest,
|
||||
JsonCharacterizationTest<MemorySourceAccessor>,
|
||||
::testing::WithParamInterface<std::pair<std::string_view, MemorySourceAccessor>>
|
||||
{};
|
||||
|
||||
TEST_P(MemorySourceAccessorJsonTest, from_json)
|
||||
{
|
||||
auto & [name, expected] = GetParam();
|
||||
/* Cannot use `readJsonTest` because need to compare `root` field of
|
||||
the source accessors for equality. */
|
||||
readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
|
||||
auto encoded = json::parse(encodedRaw);
|
||||
auto decoded = static_cast<MemorySourceAccessor>(encoded);
|
||||
ASSERT_EQ(decoded.root, expected.root);
|
||||
});
|
||||
}
|
||||
|
||||
TEST_P(MemorySourceAccessorJsonTest, to_json)
|
||||
{
|
||||
auto & [name, value] = GetParam();
|
||||
writeJsonTest(name, value);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
MemorySourceAccessorJSON,
|
||||
MemorySourceAccessorJsonTest,
|
||||
::testing::Values(
|
||||
std::pair{
|
||||
"simple",
|
||||
*memory_source_accessor::exampleSimple(),
|
||||
},
|
||||
std::pair{
|
||||
"complex",
|
||||
*memory_source_accessor::exampleComplex(),
|
||||
}));
|
||||
|
||||
} // namespace nix
|
||||
@@ -44,6 +44,7 @@ config_priv_h = configure_file(
|
||||
subdir('nix-meson-build-support/common')
|
||||
|
||||
sources = files(
|
||||
'alignment.cc',
|
||||
'archive.cc',
|
||||
'args.cc',
|
||||
'base-n.cc',
|
||||
@@ -62,6 +63,7 @@ sources = files(
|
||||
'json-utils.cc',
|
||||
'logging.cc',
|
||||
'lru-cache.cc',
|
||||
'memory-source-accessor.cc',
|
||||
'monitorfdhup.cc',
|
||||
'nix_api_util.cc',
|
||||
'nix_api_util_internal.cc',
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include <strings.h> // for strcasecmp
|
||||
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/alignment.hh"
|
||||
#include "nix/util/config-global.hh"
|
||||
#include "nix/util/posix-source-accessor.hh"
|
||||
#include "nix/util/source-path.hh"
|
||||
@@ -132,6 +133,11 @@ static void parseContents(CreateRegularFileSink & sink, Source & source)
|
||||
|
||||
sink.preallocateContents(size);
|
||||
|
||||
if (sink.skipContents) {
|
||||
source.skip(alignUp(size, 8));
|
||||
return;
|
||||
}
|
||||
|
||||
uint64_t left = size;
|
||||
std::array<char, 65536> buf;
|
||||
|
||||
@@ -166,7 +172,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath
|
||||
auto expectTag = [&](std::string_view expected) {
|
||||
auto tag = getString();
|
||||
if (tag != expected)
|
||||
throw badArchive("expected tag '%s', got '%s'", expected, tag);
|
||||
throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024));
|
||||
};
|
||||
|
||||
expectTag("(");
|
||||
|
||||
@@ -196,6 +196,8 @@ void NullFileSystemObjectSink::createRegularFile(
|
||||
void isExecutable() override {}
|
||||
} crf;
|
||||
|
||||
crf.skipContents = true;
|
||||
|
||||
// Even though `NullFileSystemObjectSink` doesn't do anything, it's important
|
||||
// that we call the function, to e.g. advance the parser using this
|
||||
// sink.
|
||||
|
||||
+40
-8
@@ -13,6 +13,7 @@
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/base-n.hh"
|
||||
#include "nix/util/base-nix-32.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
@@ -141,9 +142,13 @@ static HashFormat baseFromSize(std::string_view rest, HashAlgorithm algo)
|
||||
*
|
||||
* @param rest the string view to parse. Must not include any `<algo>(:|-)` prefix.
|
||||
*/
|
||||
static Hash parseLowLevel(std::string_view rest, HashAlgorithm algo, DecodeNamePair pair)
|
||||
static Hash parseLowLevel(
|
||||
std::string_view rest,
|
||||
HashAlgorithm algo,
|
||||
DecodeNamePair pair,
|
||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings)
|
||||
{
|
||||
Hash res{algo};
|
||||
Hash res{algo, xpSettings};
|
||||
std::string d;
|
||||
try {
|
||||
d = pair.decode(rest);
|
||||
@@ -244,9 +249,10 @@ Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo)
|
||||
return parseExplicitFormatUnprefixed(s, algo, baseFromSize(s, algo));
|
||||
}
|
||||
|
||||
Hash Hash::parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat format)
|
||||
Hash Hash::parseExplicitFormatUnprefixed(
|
||||
std::string_view s, HashAlgorithm algo, HashFormat format, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
return parseLowLevel(s, algo, baseExplicit(format));
|
||||
return parseLowLevel(s, algo, baseExplicit(format), xpSettings);
|
||||
}
|
||||
|
||||
Hash Hash::random(HashAlgorithm algo)
|
||||
@@ -446,10 +452,12 @@ std::string_view printHashFormat(HashFormat HashFormat)
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
|
||||
std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
if (s == "blake3")
|
||||
if (s == "blake3") {
|
||||
xpSettings.require(Xp::BLAKE3Hashes);
|
||||
return HashAlgorithm::BLAKE3;
|
||||
}
|
||||
if (s == "md5")
|
||||
return HashAlgorithm::MD5;
|
||||
if (s == "sha1")
|
||||
@@ -461,9 +469,9 @@ std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s)
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
HashAlgorithm parseHashAlgo(std::string_view s)
|
||||
HashAlgorithm parseHashAlgo(std::string_view s, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
auto opt_h = parseHashAlgoOpt(s);
|
||||
auto opt_h = parseHashAlgoOpt(s, xpSettings);
|
||||
if (opt_h)
|
||||
return *opt_h;
|
||||
else
|
||||
@@ -491,3 +499,27 @@ std::string_view printHashAlgo(HashAlgorithm ha)
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
using namespace nix;
|
||||
|
||||
Hash adl_serializer<Hash>::from_json(const json & json, const ExperimentalFeatureSettings & xpSettings)
|
||||
{
|
||||
auto & obj = getObject(json);
|
||||
auto algo = parseHashAlgo(getString(valueAt(obj, "algorithm")), xpSettings);
|
||||
auto format = parseHashFormat(getString(valueAt(obj, "format")));
|
||||
auto & hashS = getString(valueAt(obj, "hash"));
|
||||
return Hash::parseExplicitFormatUnprefixed(hashS, algo, format, xpSettings);
|
||||
}
|
||||
|
||||
void adl_serializer<Hash>::to_json(json & json, const Hash & hash)
|
||||
{
|
||||
json = {
|
||||
{"format", printHashFormat(HashFormat::Base64)},
|
||||
{"algorithm", printHashAlgo(hash.algo)},
|
||||
{"hash", hash.to_string(HashFormat::Base64, false)},
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <cassert>
|
||||
#include <type_traits>
|
||||
#include <cstdint>
|
||||
#include <bit>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/// Aligns val upwards to be a multiple of alignment.
|
||||
///
|
||||
/// @pre alignment must be a power of 2.
|
||||
template<typename T>
|
||||
requires std::is_unsigned_v<T>
|
||||
constexpr T alignUp(T val, unsigned alignment)
|
||||
{
|
||||
assert(std::has_single_bit(alignment) && "alignment must be a power of 2");
|
||||
T mask = ~(T{alignment} - 1u);
|
||||
return (val + alignment - 1) & mask;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,41 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <string_view>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
namespace nix {
|
||||
|
||||
static inline std::span<const std::byte> as_bytes(std::string_view sv) noexcept
|
||||
{
|
||||
return std::span<const std::byte>{
|
||||
reinterpret_cast<const std::byte *>(sv.data()),
|
||||
sv.size(),
|
||||
};
|
||||
}
|
||||
|
||||
static inline std::vector<std::byte> to_owned(std::span<const std::byte> bytes)
|
||||
{
|
||||
return std::vector<std::byte>{
|
||||
bytes.begin(),
|
||||
bytes.end(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @note this should be avoided, as arbitrary binary data in strings
|
||||
* views, while allowed, is not really proper. Generally this should
|
||||
* only be used as a stop-gap with other definitions that themselves
|
||||
* should be converted to accept `std::span<const std::byte>` or
|
||||
* similar, directly.
|
||||
*/
|
||||
static inline std::string_view to_str(std::span<const std::byte> sp)
|
||||
{
|
||||
return std::string_view{
|
||||
reinterpret_cast<const char *>(sp.data()),
|
||||
sp.size(),
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -14,6 +14,14 @@ namespace nix {
|
||||
*/
|
||||
struct CreateRegularFileSink : Sink
|
||||
{
|
||||
/**
|
||||
* If set to true, the sink will not be called with the contents
|
||||
* of the file. `preallocateContents()` will still be called to
|
||||
* convey the file size. Useful for sinks that want to efficiently
|
||||
* discard the contents of the file.
|
||||
*/
|
||||
bool skipContents = false;
|
||||
|
||||
virtual void isExecutable() = 0;
|
||||
|
||||
/**
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "nix/util/types.hh"
|
||||
#include "nix/util/serialise.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/json-impls.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -97,7 +98,11 @@ struct Hash
|
||||
* @param explicitFormat cannot be SRI, but must be one of the
|
||||
* "bases".
|
||||
*/
|
||||
static Hash parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat explicitFormat);
|
||||
static Hash parseExplicitFormatUnprefixed(
|
||||
std::string_view s,
|
||||
HashAlgorithm algo,
|
||||
HashFormat explicitFormat,
|
||||
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
|
||||
static Hash parseSRI(std::string_view original);
|
||||
|
||||
@@ -188,12 +193,14 @@ std::string_view printHashFormat(HashFormat hashFormat);
|
||||
/**
|
||||
* Parse a string representing a hash algorithm.
|
||||
*/
|
||||
HashAlgorithm parseHashAlgo(std::string_view s);
|
||||
HashAlgorithm
|
||||
parseHashAlgo(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
|
||||
/**
|
||||
* Will return nothing on parse error
|
||||
*/
|
||||
std::optional<HashAlgorithm> parseHashAlgoOpt(std::string_view s);
|
||||
std::optional<HashAlgorithm>
|
||||
parseHashAlgoOpt(std::string_view s, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
|
||||
|
||||
/**
|
||||
* And the reverse.
|
||||
@@ -221,6 +228,10 @@ public:
|
||||
HashResult currentHash();
|
||||
};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<Hash> : std::true_type
|
||||
{};
|
||||
|
||||
} // namespace nix
|
||||
|
||||
template<>
|
||||
@@ -241,3 +252,5 @@ inline std::size_t hash_value(const Hash & hash)
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
JSON_IMPL_WITH_XP_FEATURES(Hash)
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
#include "nix/util/experimental-features.hh"
|
||||
|
||||
// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
|
||||
#define JSON_IMPL(TYPE) \
|
||||
namespace nlohmann { \
|
||||
@@ -14,3 +16,15 @@
|
||||
static void to_json(json & json, const TYPE & t); \
|
||||
}; \
|
||||
}
|
||||
|
||||
#define JSON_IMPL_WITH_XP_FEATURES(TYPE) \
|
||||
namespace nlohmann { \
|
||||
using namespace nix; \
|
||||
template<> \
|
||||
struct adl_serializer<TYPE> \
|
||||
{ \
|
||||
static TYPE \
|
||||
from_json(const json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); \
|
||||
static void to_json(json & json, const TYPE & t); \
|
||||
}; \
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "nix/util/source-path.hh"
|
||||
#include "nix/util/fs-sink.hh"
|
||||
#include "nix/util/variant-wrapper.hh"
|
||||
#include "nix/util/json-impls.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -25,7 +26,7 @@ struct MemorySourceAccessor : virtual SourceAccessor
|
||||
struct Regular
|
||||
{
|
||||
bool executable = false;
|
||||
std::string contents;
|
||||
std::vector<std::byte> contents;
|
||||
|
||||
bool operator==(const Regular &) const = default;
|
||||
auto operator<=>(const Regular &) const = default;
|
||||
@@ -86,7 +87,13 @@ struct MemorySourceAccessor : virtual SourceAccessor
|
||||
*/
|
||||
File * open(const CanonPath & path, std::optional<File> create);
|
||||
|
||||
SourcePath addFile(CanonPath path, std::string && contents);
|
||||
SourcePath addFile(CanonPath path, std::vector<std::byte> && contents);
|
||||
|
||||
/**
|
||||
* Small wrapper of the other `addFile`, purely for convenience when
|
||||
* the file in question to be added is a string.
|
||||
*/
|
||||
SourcePath addFile(CanonPath path, std::string_view contents);
|
||||
};
|
||||
|
||||
inline bool MemorySourceAccessor::File::Directory::operator==(
|
||||
@@ -121,4 +128,30 @@ struct MemorySink : FileSystemObjectSink
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<MemorySourceAccessor::File::Regular> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<MemorySourceAccessor::File::Directory> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<MemorySourceAccessor::File::Symlink> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<MemorySourceAccessor::File> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<MemorySourceAccessor> : std::true_type
|
||||
{};
|
||||
|
||||
} // namespace nix
|
||||
|
||||
JSON_IMPL(MemorySourceAccessor::File::Regular)
|
||||
JSON_IMPL(MemorySourceAccessor::File::Directory)
|
||||
JSON_IMPL(MemorySourceAccessor::File::Symlink)
|
||||
JSON_IMPL(MemorySourceAccessor::File)
|
||||
JSON_IMPL(MemorySourceAccessor)
|
||||
@@ -4,6 +4,7 @@ include_dirs = [ include_directories('../..') ]
|
||||
|
||||
headers = files(
|
||||
'abstract-setting-to-json.hh',
|
||||
'alignment.hh',
|
||||
'ansicolor.hh',
|
||||
'archive.hh',
|
||||
'args.hh',
|
||||
@@ -11,6 +12,7 @@ headers = files(
|
||||
'array-from-string-literal.hh',
|
||||
'base-n.hh',
|
||||
'base-nix-32.hh',
|
||||
'bytes.hh',
|
||||
'callback.hh',
|
||||
'canon-path.hh',
|
||||
'checked-arithmetic.hh',
|
||||
|
||||
@@ -97,6 +97,8 @@ struct Source
|
||||
void drainInto(Sink & sink);
|
||||
|
||||
std::string drain();
|
||||
|
||||
virtual void skip(size_t len);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -177,6 +179,7 @@ struct FdSource : BufferedSource
|
||||
Descriptor fd;
|
||||
size_t read = 0;
|
||||
BackedStringView endOfFileError{"unexpected end-of-file"};
|
||||
bool isSeekable = true;
|
||||
|
||||
FdSource()
|
||||
: fd(INVALID_DESCRIPTOR)
|
||||
@@ -200,6 +203,8 @@ struct FdSource : BufferedSource
|
||||
*/
|
||||
bool hasData();
|
||||
|
||||
void skip(size_t len) override;
|
||||
|
||||
protected:
|
||||
size_t readUnbuffered(char * data, size_t len) override;
|
||||
private:
|
||||
|
||||
@@ -241,10 +241,4 @@ ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root);
|
||||
*/
|
||||
ref<SourceAccessor> makeUnionSourceAccessor(std::vector<ref<SourceAccessor>> && accessors);
|
||||
|
||||
/**
|
||||
* Creates a new source accessor which is confined to the subdirectory
|
||||
* of the given source accessor.
|
||||
*/
|
||||
ref<SourceAccessor> projectSubdirSourceAccessor(ref<SourceAccessor>, CanonPath subdirectory);
|
||||
|
||||
} // namespace nix
|
||||
@@ -408,6 +408,17 @@ struct VerbatimURL
|
||||
[](const ParsedURL & url) -> std::string_view { return url.scheme; }},
|
||||
raw);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last non-empty path segment from the URL.
|
||||
*
|
||||
* This is useful for extracting filenames from URLs.
|
||||
* For example, "https://example.com/path/to/file.txt?query=value"
|
||||
* returns "file.txt".
|
||||
*
|
||||
* @return The last non-empty path segment, or std::nullopt if no such segment exists.
|
||||
*/
|
||||
std::optional<std::string> lastPathSegment() const;
|
||||
};
|
||||
|
||||
std::ostream & operator<<(std::ostream & os, const VerbatimURL & url);
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
#include "nix/util/base-n.hh"
|
||||
#include "nix/util/bytes.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -58,7 +61,7 @@ std::string MemorySourceAccessor::readFile(const CanonPath & path)
|
||||
if (!f)
|
||||
throw Error("file '%s' does not exist", path);
|
||||
if (auto * r = std::get_if<File::Regular>(&f->raw))
|
||||
return r->contents;
|
||||
return std::string{to_str(r->contents)};
|
||||
else
|
||||
throw Error("file '%s' is not a regular file", path);
|
||||
}
|
||||
@@ -125,7 +128,7 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path)
|
||||
throw Error("file '%s' is not a symbolic link", path);
|
||||
}
|
||||
|
||||
SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents)
|
||||
SourcePath MemorySourceAccessor::addFile(CanonPath path, std::vector<std::byte> && contents)
|
||||
{
|
||||
// Create root directory automatically if necessary as a convenience.
|
||||
if (!root && !path.isRoot())
|
||||
@@ -142,6 +145,11 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents
|
||||
return SourcePath{ref(shared_from_this()), path};
|
||||
}
|
||||
|
||||
SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string_view contents)
|
||||
{
|
||||
return addFile(path, to_owned(as_bytes(contents)));
|
||||
}
|
||||
|
||||
using File = MemorySourceAccessor::File;
|
||||
|
||||
void MemorySink::createDirectory(const CanonPath & path)
|
||||
@@ -190,9 +198,10 @@ void CreateMemoryRegularFile::preallocateContents(uint64_t len)
|
||||
regularFile.contents.reserve(len);
|
||||
}
|
||||
|
||||
void CreateMemoryRegularFile::operator()(std::string_view data)
|
||||
void CreateMemoryRegularFile::operator()(std::string_view data_)
|
||||
{
|
||||
regularFile.contents += data;
|
||||
auto data = as_bytes(data_);
|
||||
regularFile.contents.insert(regularFile.contents.end(), data.begin(), data.end());
|
||||
}
|
||||
|
||||
void MemorySink::createSymlink(const CanonPath & path, const std::string & target)
|
||||
@@ -222,3 +231,106 @@ ref<SourceAccessor> makeEmptySourceAccessor()
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
using namespace nix;
|
||||
|
||||
MemorySourceAccessor::File::Regular adl_serializer<MemorySourceAccessor::File::Regular>::from_json(const json & json)
|
||||
{
|
||||
auto & obj = getObject(json);
|
||||
return MemorySourceAccessor::File::Regular{
|
||||
.executable = getBoolean(valueAt(obj, "executable")),
|
||||
.contents = to_owned(as_bytes(base64::decode(getString(valueAt(obj, "contents"))))),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<MemorySourceAccessor::File::Regular>::to_json(
|
||||
json & json, const MemorySourceAccessor::File::Regular & val)
|
||||
{
|
||||
json = {
|
||||
{"executable", val.executable},
|
||||
{"contents", base64::encode(val.contents)},
|
||||
};
|
||||
}
|
||||
|
||||
MemorySourceAccessor::File::Directory
|
||||
adl_serializer<MemorySourceAccessor::File::Directory>::from_json(const json & json)
|
||||
{
|
||||
auto & obj = getObject(json);
|
||||
return MemorySourceAccessor::File::Directory{
|
||||
.contents = valueAt(obj, "contents"),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<MemorySourceAccessor::File::Directory>::to_json(
|
||||
json & json, const MemorySourceAccessor::File::Directory & val)
|
||||
{
|
||||
json = {
|
||||
{"contents", val.contents},
|
||||
};
|
||||
}
|
||||
|
||||
MemorySourceAccessor::File::Symlink adl_serializer<MemorySourceAccessor::File::Symlink>::from_json(const json & json)
|
||||
{
|
||||
auto & obj = getObject(json);
|
||||
return MemorySourceAccessor::File::Symlink{
|
||||
.target = getString(valueAt(obj, "target")),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<MemorySourceAccessor::File::Symlink>::to_json(
|
||||
json & json, const MemorySourceAccessor::File::Symlink & val)
|
||||
{
|
||||
json = {
|
||||
{"target", val.target},
|
||||
};
|
||||
}
|
||||
|
||||
MemorySourceAccessor::File adl_serializer<MemorySourceAccessor::File>::from_json(const json & json)
|
||||
{
|
||||
auto & obj = getObject(json);
|
||||
auto type = getString(valueAt(obj, "type"));
|
||||
if (type == "regular")
|
||||
return static_cast<MemorySourceAccessor::File::Regular>(json);
|
||||
if (type == "directory")
|
||||
return static_cast<MemorySourceAccessor::File::Directory>(json);
|
||||
if (type == "symlink")
|
||||
return static_cast<MemorySourceAccessor::File::Symlink>(json);
|
||||
else
|
||||
throw Error("unknown type of file '%s'", type);
|
||||
}
|
||||
|
||||
void adl_serializer<MemorySourceAccessor::File>::to_json(json & json, const MemorySourceAccessor::File & val)
|
||||
{
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const MemorySourceAccessor::File::Regular & r) {
|
||||
json = r;
|
||||
json["type"] = "regular";
|
||||
},
|
||||
[&](const MemorySourceAccessor::File::Directory & d) {
|
||||
json = d;
|
||||
json["type"] = "directory";
|
||||
},
|
||||
[&](const MemorySourceAccessor::File::Symlink & s) {
|
||||
json = s;
|
||||
json["type"] = "symlink";
|
||||
},
|
||||
},
|
||||
val.raw);
|
||||
}
|
||||
|
||||
MemorySourceAccessor adl_serializer<MemorySourceAccessor>::from_json(const json & json)
|
||||
{
|
||||
MemorySourceAccessor res;
|
||||
res.root = json;
|
||||
return res;
|
||||
}
|
||||
|
||||
void adl_serializer<MemorySourceAccessor>::to_json(json & json, const MemorySourceAccessor & val)
|
||||
{
|
||||
json = val.root;
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
@@ -156,7 +156,6 @@ sources = [ config_priv_h ] + files(
|
||||
'source-accessor.cc',
|
||||
'source-path.cc',
|
||||
'strings.cc',
|
||||
'subdir-source-accessor.cc',
|
||||
'suggestions.cc',
|
||||
'tarfile.cc',
|
||||
'tee-logger.cc',
|
||||
|
||||
@@ -94,9 +94,8 @@ void Source::drainInto(Sink & sink)
|
||||
{
|
||||
std::array<char, 8192> buf;
|
||||
while (true) {
|
||||
size_t n;
|
||||
try {
|
||||
n = read(buf.data(), buf.size());
|
||||
auto n = read(buf.data(), buf.size());
|
||||
sink({buf.data(), n});
|
||||
} catch (EndOfFile &) {
|
||||
break;
|
||||
@@ -111,6 +110,16 @@ std::string Source::drain()
|
||||
return std::move(s.s);
|
||||
}
|
||||
|
||||
void Source::skip(size_t len)
|
||||
{
|
||||
std::array<char, 8192> buf;
|
||||
while (len) {
|
||||
auto n = read(buf.data(), std::min(len, buf.size()));
|
||||
assert(n <= len);
|
||||
len -= n;
|
||||
}
|
||||
}
|
||||
|
||||
size_t BufferedSource::read(char * data, size_t len)
|
||||
{
|
||||
if (!buffer)
|
||||
@@ -120,7 +129,7 @@ size_t BufferedSource::read(char * data, size_t len)
|
||||
bufPosIn = readUnbuffered(buffer.get(), bufSize);
|
||||
|
||||
/* Copy out the data in the buffer. */
|
||||
size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
|
||||
auto n = std::min(len, bufPosIn - bufPosOut);
|
||||
memcpy(data, buffer.get() + bufPosOut, n);
|
||||
bufPosOut += n;
|
||||
if (bufPosIn == bufPosOut)
|
||||
@@ -191,6 +200,39 @@ bool FdSource::hasData()
|
||||
}
|
||||
}
|
||||
|
||||
void FdSource::skip(size_t len)
|
||||
{
|
||||
/* Discard data in the buffer. */
|
||||
if (len && buffer && bufPosIn - bufPosOut) {
|
||||
if (len >= bufPosIn - bufPosOut) {
|
||||
len -= bufPosIn - bufPosOut;
|
||||
bufPosIn = bufPosOut = 0;
|
||||
} else {
|
||||
bufPosOut += len;
|
||||
len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
/* If we can, seek forward in the file to skip the rest. */
|
||||
if (isSeekable && len) {
|
||||
if (lseek(fd, len, SEEK_CUR) == -1) {
|
||||
if (errno == ESPIPE)
|
||||
isSeekable = false;
|
||||
else
|
||||
throw SysError("seeking forward in file");
|
||||
} else {
|
||||
read += len;
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Otherwise, skip by reading. */
|
||||
if (len)
|
||||
BufferedSource::skip(len);
|
||||
}
|
||||
|
||||
size_t StringSource::read(char * data, size_t len)
|
||||
{
|
||||
if (pos == s.size())
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
#include "nix/util/source-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct SubdirSourceAccessor : SourceAccessor
|
||||
{
|
||||
ref<SourceAccessor> parent;
|
||||
|
||||
CanonPath subdirectory;
|
||||
|
||||
SubdirSourceAccessor(ref<SourceAccessor> && parent, CanonPath && subdirectory)
|
||||
: parent(std::move(parent))
|
||||
, subdirectory(std::move(subdirectory))
|
||||
{
|
||||
displayPrefix.clear();
|
||||
}
|
||||
|
||||
std::string readFile(const CanonPath & path) override
|
||||
{
|
||||
return parent->readFile(subdirectory / path);
|
||||
}
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
|
||||
{
|
||||
return parent->readFile(subdirectory / path, sink, sizeCallback);
|
||||
}
|
||||
|
||||
bool pathExists(const CanonPath & path) override
|
||||
{
|
||||
return parent->pathExists(subdirectory / path);
|
||||
}
|
||||
|
||||
std::optional<Stat> maybeLstat(const CanonPath & path) override
|
||||
{
|
||||
return parent->maybeLstat(subdirectory / path);
|
||||
}
|
||||
|
||||
DirEntries readDirectory(const CanonPath & path) override
|
||||
{
|
||||
return parent->readDirectory(subdirectory / path);
|
||||
}
|
||||
|
||||
std::string readLink(const CanonPath & path) override
|
||||
{
|
||||
return parent->readLink(subdirectory / path);
|
||||
}
|
||||
|
||||
std::string showPath(const CanonPath & path) override
|
||||
{
|
||||
return displayPrefix + parent->showPath(subdirectory / path) + displaySuffix;
|
||||
}
|
||||
};
|
||||
|
||||
ref<SourceAccessor> projectSubdirSourceAccessor(ref<SourceAccessor> parent, CanonPath subdirectory)
|
||||
{
|
||||
return make_ref<SubdirSourceAccessor>(std::move(parent), std::move(subdirectory));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/canon-path.hh"
|
||||
#include "nix/util/strings-inline.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
|
||||
#include <boost/url.hpp>
|
||||
|
||||
@@ -440,4 +441,21 @@ std::ostream & operator<<(std::ostream & os, const VerbatimURL & url)
|
||||
return os;
|
||||
}
|
||||
|
||||
std::optional<std::string> VerbatimURL::lastPathSegment() const
|
||||
{
|
||||
try {
|
||||
auto parsedUrl = parsed();
|
||||
auto segments = parsedUrl.pathSegments(/*skipEmpty=*/true);
|
||||
if (std::ranges::empty(segments))
|
||||
return std::nullopt;
|
||||
return segments.back();
|
||||
} catch (BadURL &) {
|
||||
// Fall back to baseNameOf for unparsable URLs
|
||||
auto name = baseNameOf(to_string());
|
||||
if (name.empty())
|
||||
return std::nullopt;
|
||||
return std::string{name};
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -76,6 +76,8 @@ struct CmdCatNar : StoreCommand, MixCat
|
||||
void run(ref<Store> store) override
|
||||
{
|
||||
AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
|
||||
if (!fd)
|
||||
throw SysError("opening NAR file '%s'", narPath);
|
||||
auto source = FdSource{fd.get()};
|
||||
auto narAccessor = makeNarAccessor(source);
|
||||
auto listing = listNar(narAccessor, CanonPath::root, true);
|
||||
|
||||
@@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand
|
||||
{
|
||||
auto json = nlohmann::json::parse(drainFD(STDIN_FILENO));
|
||||
|
||||
auto drv = Derivation::fromJSON(json);
|
||||
auto drv = static_cast<Derivation>(json);
|
||||
|
||||
auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun);
|
||||
|
||||
|
||||
@@ -12,8 +12,7 @@ a Nix expression evaluates.
|
||||
|
||||
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
|
||||
|
||||
`nix derivation add` takes a single derivation in the following format:
|
||||
|
||||
{{#include ../../protocols/json/derivation.md}}
|
||||
`nix derivation add` takes a single derivation in the JSON format.
|
||||
See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format.
|
||||
|
||||
)""
|
||||
@@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON
|
||||
if (!drvPath.isDerivation())
|
||||
continue;
|
||||
|
||||
jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON();
|
||||
jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath);
|
||||
}
|
||||
printJSON(jsonRoot);
|
||||
}
|
||||
|
||||
@@ -48,10 +48,9 @@ By default, this command only shows top-level derivations, but with
|
||||
|
||||
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
|
||||
|
||||
`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format:
|
||||
`nix derivation show` outputs a JSON map of [store path]s to derivations in JSON format.
|
||||
See [the manual](@docroot@/protocols/json/derivation.md) for a documentation of this format.
|
||||
|
||||
[store path]: @docroot@/store/store-path.md
|
||||
|
||||
{{#include ../../protocols/json/derivation.md}}
|
||||
|
||||
)""
|
||||
+9
-4
@@ -254,10 +254,15 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
|
||||
drv.args = {store->printStorePath(getEnvShPath)};
|
||||
|
||||
/* Remove derivation checks. */
|
||||
drv.env.erase("allowedReferences");
|
||||
drv.env.erase("allowedRequisites");
|
||||
drv.env.erase("disallowedReferences");
|
||||
drv.env.erase("disallowedRequisites");
|
||||
if (drv.structuredAttrs) {
|
||||
drv.structuredAttrs->structuredAttrs.erase("outputChecks");
|
||||
} else {
|
||||
drv.env.erase("allowedReferences");
|
||||
drv.env.erase("allowedRequisites");
|
||||
drv.env.erase("disallowedReferences");
|
||||
drv.env.erase("disallowedRequisites");
|
||||
}
|
||||
|
||||
drv.env.erase("name");
|
||||
|
||||
/* Rehash and write the derivation. FIXME: would be nice to use
|
||||
|
||||
+10
-2
@@ -4,6 +4,14 @@
|
||||
|
||||
using namespace nix;
|
||||
|
||||
static FdSink getNarSink()
|
||||
{
|
||||
auto fd = getStandardOutput();
|
||||
if (isatty(fd))
|
||||
throw UsageError("refusing to write NAR to a terminal");
|
||||
return FdSink(std::move(fd));
|
||||
}
|
||||
|
||||
struct CmdDumpPath : StorePathCommand
|
||||
{
|
||||
std::string description() override
|
||||
@@ -20,7 +28,7 @@ struct CmdDumpPath : StorePathCommand
|
||||
|
||||
void run(ref<Store> store, const StorePath & storePath) override
|
||||
{
|
||||
FdSink sink(getStandardOutput());
|
||||
auto sink = getNarSink();
|
||||
store->narFromPath(storePath, sink);
|
||||
sink.flush();
|
||||
}
|
||||
@@ -51,7 +59,7 @@ struct CmdDumpPath2 : Command
|
||||
|
||||
void run() override
|
||||
{
|
||||
FdSink sink(getStandardOutput());
|
||||
auto sink = getNarSink();
|
||||
dumpPath(path, sink);
|
||||
sink.flush();
|
||||
}
|
||||
|
||||
@@ -146,6 +146,8 @@ struct CmdLsNar : Command, MixLs
|
||||
void run() override
|
||||
{
|
||||
AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
|
||||
if (!fd)
|
||||
throw SysError("opening NAR file '%s'", narPath);
|
||||
auto source = FdSource{fd.get()};
|
||||
auto narAccessor = makeNarAccessor(source);
|
||||
auto listing = listNar(narAccessor, CanonPath::root, true);
|
||||
|
||||
+15
-7
@@ -13,6 +13,8 @@
|
||||
#include "nix/cmd/misc-store-flags.hh"
|
||||
#include "nix/util/terminal.hh"
|
||||
#include "nix/util/environment-variables.hh"
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/store/path.hh"
|
||||
|
||||
#include "man-pages.hh"
|
||||
|
||||
@@ -56,7 +58,7 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url)
|
||||
|
||||
std::tuple<StorePath, Hash> prefetchFile(
|
||||
ref<Store> store,
|
||||
std::string_view url,
|
||||
const VerbatimURL & url,
|
||||
std::optional<std::string> name,
|
||||
HashAlgorithm hashAlgo,
|
||||
std::optional<Hash> expectedHash,
|
||||
@@ -68,9 +70,15 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||
|
||||
/* Figure out a name in the Nix store. */
|
||||
if (!name) {
|
||||
name = baseNameOf(url);
|
||||
if (name->empty())
|
||||
throw Error("cannot figure out file name for '%s'", url);
|
||||
name = url.lastPathSegment();
|
||||
if (!name || name->empty())
|
||||
throw Error("cannot figure out file name for '%s'", url.to_string());
|
||||
}
|
||||
try {
|
||||
checkName(*name);
|
||||
} catch (BadStorePathName & e) {
|
||||
e.addTrace({}, "file name '%s' was extracted from URL '%s'", *name, url.to_string());
|
||||
throw;
|
||||
}
|
||||
|
||||
std::optional<StorePath> storePath;
|
||||
@@ -105,14 +113,14 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||
|
||||
FdSink sink(fd.get());
|
||||
|
||||
FileTransferRequest req(VerbatimURL{url});
|
||||
FileTransferRequest req(url);
|
||||
req.decompress = false;
|
||||
getFileTransfer()->download(std::move(req), sink);
|
||||
}
|
||||
|
||||
/* Optionally unpack the file. */
|
||||
if (unpack) {
|
||||
Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url));
|
||||
Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url.to_string()));
|
||||
auto unpacked = (tmpDir.path() / "unpacked").string();
|
||||
createDirs(unpacked);
|
||||
unpackTarfile(tmpFile.string(), unpacked);
|
||||
@@ -128,7 +136,7 @@ std::tuple<StorePath, Hash> prefetchFile(
|
||||
}
|
||||
}
|
||||
|
||||
Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url));
|
||||
Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string()));
|
||||
|
||||
auto info = store->addToStoreSlow(
|
||||
*name, PosixSourceAccessor::createAtRoot(tmpFile), method, hashAlgo, {}, expectedHash);
|
||||
|
||||
@@ -18,6 +18,21 @@ cat <<EOF >"$TEST_HOME/flake.nix"
|
||||
outputs = [ "out" "dev" ];
|
||||
meta.outputsToInstall = [ "out" ];
|
||||
buildCommand = "";
|
||||
# ensure we're stripping these from the environment derivation
|
||||
disallowedReferences = [ "out" ];
|
||||
disallowedRequisites = [ "out" ];
|
||||
};
|
||||
packages.$system.hello-structured = (import ./config.nix).mkDerivation {
|
||||
__structuredAttrs = true;
|
||||
name = "hello";
|
||||
outputs = [ "out" "dev" ];
|
||||
meta.outputsToInstall = [ "out" ];
|
||||
buildCommand = "";
|
||||
# ensure we're stripping these from the environment derivation
|
||||
outputChecks.out = {
|
||||
disallowedReferences = [ "out" ];
|
||||
disallowedRequisites = [ "out" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -142,4 +157,7 @@ echo "\$SHELL"
|
||||
EOF
|
||||
)" -ef "$BASH_INTERACTIVE_EXECUTABLE" ]]
|
||||
|
||||
# Test whether `nix develop` works with `__structuredAttrs`
|
||||
[[ -z "$(nix develop --no-write-lock-file .#hello-structured </dev/null)" ]]
|
||||
|
||||
clearStore
|
||||
@@ -25,6 +25,13 @@ import $testDir/undefined-variable.nix
|
||||
|
||||
TODO_NixOS
|
||||
|
||||
# FIXME: repl tests fail on systems with stack limits
|
||||
stack_ulimit="$(ulimit -Hs)"
|
||||
stack_required="$((64 * 1024 * 1024))"
|
||||
if [[ "$stack_ulimit" != "unlimited" ]]; then
|
||||
((stack_ulimit < stack_required)) && skipTest "repl tests cannot run on systems with stack size <$stack_required ($stack_ulimit)"
|
||||
fi
|
||||
|
||||
testRepl () {
|
||||
local nixArgs
|
||||
nixArgs=("$@")
|
||||
|
||||
@@ -83,9 +83,11 @@ in
|
||||
ENDPOINT = 'http://server:9000'
|
||||
REGION = 'eu-west-1'
|
||||
|
||||
PKG_A = '${pkgA}'
|
||||
PKG_B = '${pkgB}'
|
||||
PKG_C = '${pkgC}'
|
||||
PKGS = {
|
||||
'A': '${pkgA}',
|
||||
'B': '${pkgB}',
|
||||
'C': '${pkgC}',
|
||||
}
|
||||
|
||||
ENV_WITH_CREDS = f"AWS_ACCESS_KEY_ID={ACCESS_KEY} AWS_SECRET_ACCESS_KEY={SECRET_KEY}"
|
||||
|
||||
@@ -101,10 +103,6 @@ in
|
||||
bucket_and_path = f"{bucket}{path}" if path else bucket
|
||||
return f"s3://{bucket_and_path}?{query}"
|
||||
|
||||
def make_http_url(path):
|
||||
"""Build HTTP URL for direct S3 access"""
|
||||
return f"{ENDPOINT}/{path}"
|
||||
|
||||
def get_package_hash(pkg_path):
|
||||
"""Extract store hash from package path"""
|
||||
return pkg_path.split("/")[-1].split("-")[0]
|
||||
@@ -133,26 +131,49 @@ in
|
||||
print(output)
|
||||
raise Exception(f"{error_msg}: expected {expected}, got {actual}")
|
||||
|
||||
def with_test_bucket(populate_with=[]):
|
||||
def verify_packages_in_store(machine, pkg_paths, should_exist=True):
|
||||
"""
|
||||
Verify whether packages exist in the store.
|
||||
|
||||
Args:
|
||||
machine: The machine to check on
|
||||
pkg_paths: List of package paths to check (or single path)
|
||||
should_exist: If True, verify packages exist; if False, verify they don't
|
||||
"""
|
||||
paths = [pkg_paths] if isinstance(pkg_paths, str) else pkg_paths
|
||||
for pkg in paths:
|
||||
if should_exist:
|
||||
machine.succeed(f"nix path-info {pkg}")
|
||||
else:
|
||||
machine.fail(f"nix path-info {pkg}")
|
||||
|
||||
def setup_s3(populate_bucket=[], public=False):
|
||||
"""
|
||||
Decorator that creates/destroys a unique bucket for each test.
|
||||
Optionally pre-populates bucket with specified packages.
|
||||
Cleans up client store after test completion.
|
||||
|
||||
Args:
|
||||
populate_with: List of packages to upload before test runs
|
||||
populate_bucket: List of packages to upload before test runs
|
||||
public: If True, make the bucket publicly accessible
|
||||
"""
|
||||
def decorator(test_func):
|
||||
def wrapper():
|
||||
bucket = str(uuid.uuid4())
|
||||
server.succeed(f"mc mb minio/{bucket}")
|
||||
if public:
|
||||
server.succeed(f"mc anonymous set download minio/{bucket}")
|
||||
try:
|
||||
if populate_with:
|
||||
if populate_bucket:
|
||||
store_url = make_s3_url(bucket)
|
||||
for pkg in populate_with:
|
||||
for pkg in populate_bucket:
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {pkg}")
|
||||
test_func(bucket)
|
||||
finally:
|
||||
server.succeed(f"mc rb --force minio/{bucket}")
|
||||
# Clean up client store - only delete if path exists
|
||||
for pkg in PKGS.values():
|
||||
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
@@ -160,7 +181,7 @@ in
|
||||
# Test Functions
|
||||
# ============================================================================
|
||||
|
||||
@with_test_bucket()
|
||||
@setup_s3()
|
||||
def test_credential_caching(bucket):
|
||||
"""Verify credential providers are cached and reused"""
|
||||
print("\n=== Testing Credential Caching ===")
|
||||
@@ -168,7 +189,7 @@ in
|
||||
store_url = make_s3_url(bucket)
|
||||
output = server.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' "
|
||||
f"{PKG_A} {PKG_B} {PKG_C} 2>&1"
|
||||
f"{PKGS['A']} {PKGS['B']} {PKGS['C']} 2>&1"
|
||||
)
|
||||
|
||||
assert_count(
|
||||
@@ -180,7 +201,7 @@ in
|
||||
|
||||
print("✓ Credential provider created once and cached")
|
||||
|
||||
@with_test_bucket(populate_with=[PKG_A])
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_fetchurl_basic(bucket):
|
||||
"""Test builtins.fetchurl works with s3:// URLs"""
|
||||
print("\n=== Testing builtins.fetchurl ===")
|
||||
@@ -196,13 +217,13 @@ in
|
||||
|
||||
print("✓ builtins.fetchurl works with s3:// URLs")
|
||||
|
||||
@with_test_bucket()
|
||||
@setup_s3()
|
||||
def test_error_message_formatting(bucket):
|
||||
"""Verify error messages display URLs correctly"""
|
||||
print("\n=== Testing Error Message Formatting ===")
|
||||
|
||||
nonexistent_url = make_s3_url(bucket, path="/foo-that-does-not-exist")
|
||||
expected_http_url = make_http_url(f"{bucket}/foo-that-does-not-exist")
|
||||
expected_http_url = f"{ENDPOINT}/{bucket}/foo-that-does-not-exist"
|
||||
|
||||
error_msg = client.fail(
|
||||
f"{ENV_WITH_CREDS} nix eval --impure --expr "
|
||||
@@ -216,7 +237,7 @@ in
|
||||
|
||||
print("✓ Error messages format URLs correctly")
|
||||
|
||||
@with_test_bucket(populate_with=[PKG_A])
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_fork_credential_preresolution(bucket):
|
||||
"""Test credential pre-resolution in forked processes"""
|
||||
print("\n=== Testing Fork Credential Pre-resolution ===")
|
||||
@@ -246,7 +267,7 @@ in
|
||||
""".format(id=test_id, url=test_url, hash=cache_info_hash)
|
||||
|
||||
output = client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix build --debug --impure --expr '{fetchurl_expr}' 2>&1"
|
||||
f"{ENV_WITH_CREDS} nix build --debug --impure --no-link --expr '{fetchurl_expr}' 2>&1"
|
||||
)
|
||||
|
||||
# Verify fork behavior
|
||||
@@ -296,7 +317,7 @@ in
|
||||
|
||||
print(" ✓ Child uses pre-resolved credentials (no new providers)")
|
||||
|
||||
@with_test_bucket(populate_with=[PKG_A, PKG_B, PKG_C])
|
||||
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B'], PKGS['C']])
|
||||
def test_store_operations(bucket):
|
||||
"""Test nix store info and copy operations"""
|
||||
print("\n=== Testing Store Operations ===")
|
||||
@@ -316,11 +337,11 @@ in
|
||||
print(f" ✓ Store URL: {store_info['url']}")
|
||||
|
||||
# Test copy from store
|
||||
client.fail(f"nix path-info {PKG_A}")
|
||||
verify_packages_in_store(client, PKGS['A'], should_exist=False)
|
||||
|
||||
output = client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix copy --debug --no-check-sigs "
|
||||
f"--from '{store_url}' {PKG_A} {PKG_B} {PKG_C} 2>&1"
|
||||
f"--from '{store_url}' {PKGS['A']} {PKGS['B']} {PKGS['C']} 2>&1"
|
||||
)
|
||||
|
||||
assert_count(
|
||||
@@ -330,12 +351,46 @@ in
|
||||
"Client credential provider caching failed"
|
||||
)
|
||||
|
||||
client.succeed(f"nix path-info {PKG_A}")
|
||||
verify_packages_in_store(client, [PKGS['A'], PKGS['B'], PKGS['C']])
|
||||
|
||||
print(" ✓ nix copy works")
|
||||
print(" ✓ Credentials cached on client")
|
||||
|
||||
@with_test_bucket(populate_with=[PKG_A])
|
||||
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B']], public=True)
|
||||
def test_public_bucket_operations(bucket):
|
||||
"""Test store operations on public bucket without credentials"""
|
||||
print("\n=== Testing Public Bucket Operations ===")
|
||||
|
||||
store_url = make_s3_url(bucket)
|
||||
|
||||
# Verify store info works without credentials
|
||||
client.succeed(f"nix store info --store '{store_url}' >&2")
|
||||
print(" ✓ nix store info works without credentials")
|
||||
|
||||
# Get and validate store info JSON
|
||||
info_json = client.succeed(f"nix store info --json --store '{store_url}'")
|
||||
store_info = json.loads(info_json)
|
||||
|
||||
if not store_info.get("url"):
|
||||
raise Exception("Store should have a URL")
|
||||
|
||||
print(f" ✓ Store URL: {store_info['url']}")
|
||||
|
||||
# Verify packages are not yet in client store
|
||||
verify_packages_in_store(client, [PKGS['A'], PKGS['B']], should_exist=False)
|
||||
|
||||
# Test copy from public bucket without credentials
|
||||
client.succeed(
|
||||
f"nix copy --debug --no-check-sigs "
|
||||
f"--from '{store_url}' {PKGS['A']} {PKGS['B']} 2>&1"
|
||||
)
|
||||
|
||||
# Verify packages were copied successfully
|
||||
verify_packages_in_store(client, [PKGS['A'], PKGS['B']])
|
||||
|
||||
print(" ✓ nix copy from public bucket works without credentials")
|
||||
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_url_format_variations(bucket):
|
||||
"""Test different S3 URL parameter combinations"""
|
||||
print("\n=== Testing URL Format Variations ===")
|
||||
@@ -350,7 +405,7 @@ in
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2")
|
||||
print(" ✓ Parameter order: endpoint before region works")
|
||||
|
||||
@with_test_bucket(populate_with=[PKG_A])
|
||||
@setup_s3(populate_bucket=[PKGS['A']])
|
||||
def test_concurrent_fetches(bucket):
|
||||
"""Validate thread safety with concurrent S3 operations"""
|
||||
print("\n=== Testing Concurrent Fetches ===")
|
||||
@@ -386,12 +441,12 @@ in
|
||||
|
||||
try:
|
||||
output = client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix build --debug --impure "
|
||||
f"{ENV_WITH_CREDS} nix build --debug --impure --no-link "
|
||||
f"--expr '{concurrent_expr}' --max-jobs 5 2>&1"
|
||||
)
|
||||
except:
|
||||
output = client.fail(
|
||||
f"{ENV_WITH_CREDS} nix build --debug --impure "
|
||||
f"{ENV_WITH_CREDS} nix build --debug --impure --no-link "
|
||||
f"--expr '{concurrent_expr}' --max-jobs 5 2>&1"
|
||||
)
|
||||
|
||||
@@ -412,26 +467,33 @@ in
|
||||
f"Expected 5 FileTransfer instances for 5 concurrent fetches, got {transfers_created}"
|
||||
)
|
||||
|
||||
@with_test_bucket()
|
||||
if providers_created != 1:
|
||||
print("Debug output:")
|
||||
print(output)
|
||||
raise Exception(
|
||||
f"Expected 1 credential provider for concurrent fetches, got {providers_created}"
|
||||
)
|
||||
|
||||
@setup_s3()
|
||||
def test_compression_narinfo_gzip(bucket):
|
||||
"""Test narinfo compression with gzip"""
|
||||
print("\n=== Testing Compression: narinfo (gzip) ===")
|
||||
|
||||
store_url = make_s3_url(bucket, **{'narinfo-compression': 'gzip'})
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_B}")
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['B']}")
|
||||
|
||||
pkg_hash = get_package_hash(PKG_B)
|
||||
pkg_hash = get_package_hash(PKGS['B'])
|
||||
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip")
|
||||
|
||||
print(" ✓ .narinfo has Content-Encoding: gzip")
|
||||
|
||||
# Verify client can download and decompress
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_B}")
|
||||
client.succeed(f"nix path-info {PKG_B}")
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}")
|
||||
verify_packages_in_store(client, PKGS['B'])
|
||||
|
||||
print(" ✓ Client decompressed .narinfo successfully")
|
||||
|
||||
@with_test_bucket()
|
||||
@setup_s3()
|
||||
def test_compression_mixed(bucket):
|
||||
"""Test mixed compression (narinfo=xz, ls=gzip)"""
|
||||
print("\n=== Testing Compression: mixed (narinfo=xz, ls=gzip) ===")
|
||||
@@ -441,9 +503,9 @@ in
|
||||
**{'narinfo-compression': 'xz', 'write-nar-listing': 'true', 'ls-compression': 'gzip'}
|
||||
)
|
||||
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_C}")
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['C']}")
|
||||
|
||||
pkg_hash = get_package_hash(PKG_C)
|
||||
pkg_hash = get_package_hash(PKGS['C'])
|
||||
|
||||
# Verify .narinfo has xz compression
|
||||
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz")
|
||||
@@ -454,24 +516,87 @@ in
|
||||
print(" ✓ .ls has Content-Encoding: gzip")
|
||||
|
||||
# Verify client can download with mixed compression
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKG_C}")
|
||||
client.succeed(f"nix path-info {PKG_C}")
|
||||
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}")
|
||||
verify_packages_in_store(client, PKGS['C'])
|
||||
|
||||
print(" ✓ Client downloaded package with mixed compression")
|
||||
|
||||
@with_test_bucket()
|
||||
@setup_s3()
|
||||
def test_compression_disabled(bucket):
|
||||
"""Verify no compression by default"""
|
||||
print("\n=== Testing Compression: disabled (default) ===")
|
||||
|
||||
store_url = make_s3_url(bucket)
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKG_A}")
|
||||
server.succeed(f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']}")
|
||||
|
||||
pkg_hash = get_package_hash(PKG_A)
|
||||
pkg_hash = get_package_hash(PKGS['A'])
|
||||
verify_no_compression(server, bucket, f"{pkg_hash}.narinfo")
|
||||
|
||||
print(" ✓ No compression applied by default")
|
||||
|
||||
@setup_s3()
|
||||
def test_nix_prefetch_url(bucket):
|
||||
"""Test that nix-prefetch-url retrieves actual file content from S3, not empty files (issue #8862)"""
|
||||
print("\n=== Testing nix-prefetch-url S3 Content Retrieval (issue #8862) ===")
|
||||
|
||||
# Create a test file with known content
|
||||
test_content = "This is test content to verify S3 downloads work correctly!\n"
|
||||
test_file_size = len(test_content)
|
||||
|
||||
server.succeed(f"echo -n '{test_content}' > /tmp/test-file.txt")
|
||||
|
||||
# Upload to S3
|
||||
server.succeed(f"mc cp /tmp/test-file.txt minio/{bucket}/test-file.txt")
|
||||
|
||||
# Calculate expected hash
|
||||
expected_hash = server.succeed(
|
||||
"nix hash file --type sha256 --base32 /tmp/test-file.txt"
|
||||
).strip()
|
||||
|
||||
print(f" ✓ Uploaded test file to S3 ({test_file_size} bytes)")
|
||||
|
||||
# Use nix-prefetch-url to download from S3
|
||||
s3_url = make_s3_url(bucket, path="/test-file.txt")
|
||||
|
||||
prefetch_output = client.succeed(
|
||||
f"{ENV_WITH_CREDS} nix-prefetch-url --print-path '{s3_url}'"
|
||||
)
|
||||
|
||||
# Extract hash and store path
|
||||
# With --print-path, output is: <hash>\n<store-path>
|
||||
lines = prefetch_output.strip().split('\n')
|
||||
prefetch_hash = lines[0] # First line is the hash
|
||||
store_path = lines[1] # Second line is the store path
|
||||
|
||||
# Verify hash matches
|
||||
if prefetch_hash != expected_hash:
|
||||
raise Exception(
|
||||
f"Hash mismatch: expected {expected_hash}, got {prefetch_hash}"
|
||||
)
|
||||
|
||||
print(" ✓ nix-prefetch-url completed with correct hash")
|
||||
|
||||
# Verify the downloaded file is NOT empty (the bug in #8862)
|
||||
file_size = int(client.succeed(f"stat -c %s {store_path}").strip())
|
||||
|
||||
if file_size == 0:
|
||||
raise Exception("Downloaded file is EMPTY - issue #8862 regression detected!")
|
||||
|
||||
if file_size != test_file_size:
|
||||
raise Exception(
|
||||
f"File size mismatch: expected {test_file_size}, got {file_size}"
|
||||
)
|
||||
|
||||
print(f" ✓ File has correct size ({file_size} bytes, not empty)")
|
||||
|
||||
# Verify actual content matches by comparing hashes instead of printing entire file
|
||||
downloaded_hash = client.succeed(f"nix hash file --type sha256 --base32 {store_path}").strip()
|
||||
|
||||
if downloaded_hash != expected_hash:
|
||||
raise Exception(f"Content hash mismatch: expected {expected_hash}, got {downloaded_hash}")
|
||||
|
||||
print(" ✓ File content verified correct (hash matches)")
|
||||
|
||||
# ============================================================================
|
||||
# Main Test Execution
|
||||
# ============================================================================
|
||||
@@ -494,11 +619,13 @@ in
|
||||
test_error_message_formatting()
|
||||
test_fork_credential_preresolution()
|
||||
test_store_operations()
|
||||
test_public_bucket_operations()
|
||||
test_url_format_variations()
|
||||
test_concurrent_fetches()
|
||||
test_compression_narinfo_gzip()
|
||||
test_compression_mixed()
|
||||
test_compression_disabled()
|
||||
test_nix_prefetch_url()
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("✓ All S3 Binary Cache Store Tests Passed!")
|
||||
|
||||
Reference in new issue
Block a user