Compare commits
141
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a565e173bf | ||
|
|
00f4a860e7 | ||
|
|
da637a05da | ||
|
|
956fffdd6f | ||
|
|
bac41d6989 | ||
|
|
de192794c9 | ||
|
|
246dbe1c05 | ||
|
|
194c21fc82 | ||
|
|
e08853a67c | ||
|
|
ae49074548 | ||
|
|
f1d4fab1e5 | ||
|
|
c874e7071b | ||
|
|
c67966418f | ||
|
|
be2572ed8d | ||
|
|
be99a1c6bb | ||
|
|
fe8cdbc3e4 | ||
|
|
70176ed317 | ||
|
|
84a5bee424 | ||
|
|
e3246301a6 | ||
|
|
d4c69c7b8f | ||
|
|
f5aafbd6ed | ||
|
|
1a4ad0706b | ||
|
|
972915cabd | ||
|
|
dd0d006517 | ||
|
|
1d3f0ca22e | ||
|
|
c592090fff | ||
|
|
4b6d07d642 | ||
|
|
e177f42536 | ||
|
|
ac8b1efcf9 | ||
|
|
aa4106fd68 | ||
|
|
7f1d92793e | ||
|
|
234f029940 | ||
|
|
dd716dc9be | ||
|
|
ea17cc1b57 | ||
|
|
0c1be3aabe | ||
|
|
1c41e07b46 | ||
|
|
ad664ce64e | ||
|
|
6ca3434cac | ||
|
|
6129aee988 | ||
|
|
5e220271e2 | ||
|
|
8e6b69de54 | ||
|
|
3915b3a111 | ||
|
|
c5515bb22e | ||
|
|
91b69e9e70 | ||
|
|
9e9dfe36df | ||
|
|
50e8d17f3c | ||
|
|
ef8dd58d9b | ||
|
|
91ed3701fe | ||
|
|
b8e5d1f290 | ||
|
|
d44b33562f | ||
|
|
d46504a136 | ||
|
|
126f30deb2 | ||
|
|
5dcfa86910 | ||
|
|
6b6ceddf72 | ||
|
|
60f9489b83 | ||
|
|
584a8e8a00 | ||
|
|
f234633e27 | ||
|
|
6417863ce9 | ||
|
|
91cd42511e | ||
|
|
1af5a98955 | ||
|
|
17777e3b70 | ||
|
|
9321669353 | ||
|
|
3742ae061e | ||
|
|
a91115bf22 | ||
|
|
8c8b706f6b | ||
|
|
fb26285458 | ||
|
|
bbfaaf3a20 | ||
|
|
f9b73185e4 | ||
|
|
27e3d28ed8 | ||
|
|
3994e5627f | ||
|
|
ec2fd2dc23 | ||
|
|
fdc5600fa7 | ||
|
|
1f6ac88efc | ||
|
|
9d4d10954a | ||
|
|
7e53afd8b9 | ||
|
|
bef3c37cb2 | ||
|
|
0f0d9255c6 | ||
|
|
f0b95b6d5b | ||
|
|
e38128b90d | ||
|
|
78e98691d6 | ||
|
|
e213fd64b6 | ||
|
|
1cd8458c28 | ||
|
|
ecaf9470b9 | ||
|
|
8b7e03f0f9 | ||
|
|
04606d50d1 | ||
|
|
476c21d5ef | ||
|
|
1a9ba0d6fe | ||
|
|
648714cd44 | ||
|
|
6b7223b6b7 | ||
|
|
afe5ed879f | ||
|
|
d924374bf2 | ||
|
|
f1968ea38e | ||
|
|
8d338c9234 | ||
|
|
9a695f9067 | ||
|
|
925c0fa4a2 | ||
|
|
7308fde0bc | ||
|
|
324bfd82dc | ||
|
|
8e01e4ad5c | ||
|
|
4c4eb5d07f | ||
|
|
b5ae3e10c2 | ||
|
|
4f5af471fb | ||
|
|
b9af19cedf | ||
|
|
d6f1e2de21 | ||
|
|
5d365cd61f | ||
|
|
c87f29a0b6 | ||
|
|
f594a8e11e | ||
|
|
0a74b4905c | ||
|
|
d74177dccc | ||
|
|
36ee38efd1 | ||
|
|
5d7912eb18 | ||
|
|
78888ec8a8 | ||
|
|
b047cecf5c | ||
|
|
d0217ec180 | ||
|
|
953929f899 | ||
|
|
3c83856494 | ||
|
|
f3d8d1f719 | ||
|
|
c8a15bf70d | ||
|
|
ad5c6a53b9 | ||
|
|
350d602832 | ||
|
|
115dea10b2 | ||
|
|
ddb8830c97 | ||
|
|
459f9e0185 | ||
|
|
5390bba920 | ||
|
|
387eceff45 | ||
|
|
96c8cc550f | ||
|
|
32b286e5d6 | ||
|
|
b558dac7a9 | ||
|
|
7e8db2eb59 | ||
|
|
6ca2efc7d4 | ||
|
|
62247af363 | ||
|
|
a4a49a9dae | ||
|
|
606c258c6f | ||
|
|
3d147c04a5 | ||
|
|
61fbef42a6 | ||
|
|
6036aaf798 | ||
|
|
a9d9b50b72 | ||
|
|
6fa03765ed | ||
|
|
12293a8b11 | ||
|
|
3fb943d130 | ||
|
|
aace1fb5d6 | ||
|
|
9abcc68ad1 |
No files matched your search
@@ -0,0 +1,14 @@
|
||||
# Disable CodeRabbit auto-review to prevent verbose comments on PRs.
|
||||
# When enabled: false, CodeRabbit won't attempt reviews and won't post
|
||||
# "Review skipped" or other automated comments.
|
||||
reviews:
|
||||
auto_review:
|
||||
enabled: false
|
||||
review_status: false
|
||||
high_level_summary: false
|
||||
poem: false
|
||||
sequence_diagrams: false
|
||||
changed_files_summary: false
|
||||
tools:
|
||||
github-checks:
|
||||
enabled: false
|
||||
@@ -16,13 +16,17 @@ inputs:
|
||||
install_url:
|
||||
description: "URL of the Nix installer"
|
||||
required: false
|
||||
default: "https://releases.nixos.org/nix/nix-2.30.2/install"
|
||||
default: "https://releases.nixos.org/nix/nix-2.32.1/install"
|
||||
tarball_url:
|
||||
description: "URL of the Nix tarball to use with the experimental installer"
|
||||
required: false
|
||||
github_token:
|
||||
description: "Github token"
|
||||
required: true
|
||||
use_cache:
|
||||
description: "Whether to setup magic-nix-cache"
|
||||
default: true
|
||||
required: false
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
@@ -118,3 +122,10 @@ runs:
|
||||
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
|
||||
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
|
||||
extra-conf: ${{ inputs.extra_nix_config }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
|
||||
if: ${{ inputs.use_cache == 'true' }}
|
||||
with:
|
||||
diagnostic-endpoint: ''
|
||||
use-flakehub: false
|
||||
use-gha-cache: true
|
||||
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
|
||||
+12
-10
@@ -14,6 +14,10 @@ on:
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
@@ -29,6 +33,7 @@ jobs:
|
||||
extra_nix_config:
|
||||
experimental-features = nix-command flakes
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
use_cache: false
|
||||
- run: nix flake show --all-systems --json
|
||||
|
||||
pre-commit-checks:
|
||||
@@ -41,7 +46,6 @@ jobs:
|
||||
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
|
||||
extra_nix_config: experimental-features = nix-command flakes
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@main
|
||||
- run: ./ci/gha/tests/pre-commit-checks
|
||||
|
||||
basic-checks:
|
||||
@@ -92,7 +96,6 @@ jobs:
|
||||
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
|
||||
# The sandbox would otherwise be disabled by default on Darwin
|
||||
extra_nix_config: "sandbox = true"
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@main
|
||||
# Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user:
|
||||
# https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces
|
||||
- run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
|
||||
@@ -171,7 +174,7 @@ jobs:
|
||||
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
||||
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
|
||||
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
|
||||
- uses: cachix/install-nix-action@v31
|
||||
- uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1
|
||||
if: ${{ !matrix.experimental-installer }}
|
||||
with:
|
||||
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
@@ -227,12 +230,13 @@ jobs:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: cachix/install-nix-action@v31
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
install_url: https://releases.nixos.org/nix/nix-2.20.3/install
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@main
|
||||
- run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
|
||||
- run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
|
||||
dogfood: false
|
||||
extra_nix_config: |
|
||||
experimental-features = flakes nix-command
|
||||
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
|
||||
- run: nix build .#dockerImage -L
|
||||
- run: docker load -i ./result/image.tar.gz
|
||||
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
|
||||
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
|
||||
@@ -289,7 +293,6 @@ jobs:
|
||||
extra_nix_config:
|
||||
experimental-features = nix-command flakes
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@main
|
||||
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
|
||||
|
||||
profile_build:
|
||||
@@ -310,7 +313,6 @@ jobs:
|
||||
extra_nix_config: |
|
||||
experimental-features = flakes nix-command ca-derivations impure-derivations
|
||||
max-jobs = 1
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@main
|
||||
- run: |
|
||||
nix build -L --file ./ci/gha/profile-build buildTimeReport --out-link build-time-report.md
|
||||
cat build-time-report.md >> $GITHUB_STEP_SUMMARY
|
||||
@@ -0,0 +1,35 @@
|
||||
name: "Coverity Scan"
|
||||
|
||||
on:
|
||||
# FIXME: This is here for testing, should be removed before merging
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- coverity-test
|
||||
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
coverity:
|
||||
name: Coverity static analysis
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
dogfood: false
|
||||
extra_nix_config: experimental-features = nix-command flakes
|
||||
- name: configure
|
||||
run: nix develop --configure
|
||||
- uses: vapier/coverity-scan-action@v1
|
||||
with:
|
||||
project: NixOS/nix
|
||||
email: ${{ secrets.COVERITY_SCAN_EMAIL }}
|
||||
token: ${{ secrets.COVERITY_SCAN_TOKEN }}
|
||||
build_language: cxx
|
||||
command: nix develop --build
|
||||
version: ${{ github.sha }}
|
||||
@@ -35,6 +35,9 @@ mkMesonDerivation (finalAttrs: {
|
||||
../../.version
|
||||
# For example JSON
|
||||
../../src/libutil-tests/data/hash
|
||||
../../src/libstore-tests/data/content-address
|
||||
../../src/libstore-tests/data/store-path
|
||||
../../src/libstore-tests/data/derived-path
|
||||
# Too many different types of files to filter for now
|
||||
../../doc/manual
|
||||
./.
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
synopsis: "S3 URLs now support object versioning via versionId parameter"
|
||||
prs: [14274]
|
||||
issues: [13955]
|
||||
---
|
||||
|
||||
S3 URLs now support a `versionId` query parameter to fetch specific versions
|
||||
of objects from S3 buckets with versioning enabled. This allows pinning to
|
||||
exact object versions for reproducibility and protection against unexpected
|
||||
changes:
|
||||
|
||||
```
|
||||
s3://bucket/key?region=us-east-1&versionId=abc123def456
|
||||
```
|
||||
@@ -118,8 +118,11 @@
|
||||
- [Formats and Protocols](protocols/index.md)
|
||||
- [JSON Formats](protocols/json/index.md)
|
||||
- [Hash](protocols/json/hash.md)
|
||||
- [Content Address](protocols/json/content-address.md)
|
||||
- [Store Path](protocols/json/store-path.md)
|
||||
- [Store Object Info](protocols/json/store-object-info.md)
|
||||
- [Derivation](protocols/json/derivation.md)
|
||||
- [Deriving Path](protocols/json/deriving-path.md)
|
||||
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
|
||||
- [Store Path Specification](protocols/store-path.md)
|
||||
- [Nix Archive (NAR) Format](protocols/nix-archive.md)
|
||||
|
||||
@@ -48,8 +48,7 @@ The behaviour of the collector is also influenced by the
|
||||
configuration file.
|
||||
|
||||
By default, the collector prints the total number of freed bytes when it
|
||||
finishes (or when it is interrupted). With `--print-dead`, it prints the
|
||||
number of bytes that would be freed.
|
||||
finishes (or when it is interrupted).
|
||||
|
||||
{{#include ./opt-common.md}}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{{#include content-address-v1-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### [Text](@docroot@/store/store-object/content-address.html#method-text) method
|
||||
|
||||
```json
|
||||
{{#include schema/content-address-v1/text.json}}
|
||||
```
|
||||
|
||||
### [Nix Archive](@docroot@/store/store-object/content-address.html#method-nix-archive) method
|
||||
|
||||
```json
|
||||
{{#include schema/content-address-v1/nar.json}}
|
||||
```
|
||||
|
||||
<!-- need to convert YAML to JSON first
|
||||
## Raw Schema
|
||||
|
||||
[JSON Schema for Hash v1](schema/content-address-v1.json)
|
||||
-->
|
||||
@@ -0,0 +1,21 @@
|
||||
{{#include deriving-path-v1-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### Constant
|
||||
|
||||
```json
|
||||
{{#include schema/deriving-path-v1/single_opaque.json}}
|
||||
```
|
||||
|
||||
### Output of static derivation
|
||||
|
||||
```json
|
||||
{{#include schema/deriving-path-v1/single_built.json}}
|
||||
```
|
||||
|
||||
### Output of dynamic derivation
|
||||
|
||||
```json
|
||||
{{#include schema/deriving-path-v1/single_built_built.json}}
|
||||
```
|
||||
@@ -12,3 +12,6 @@ s/\\`/`/g
|
||||
# As we have more such relative links, more replacements of this nature
|
||||
# should appear below.
|
||||
s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
|
||||
s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
|
||||
s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
|
||||
s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g
|
||||
@@ -10,7 +10,10 @@ json_schema_config = files('json-schema-for-humans-config.yaml')
|
||||
|
||||
schemas = [
|
||||
'hash-v1',
|
||||
'content-address-v1',
|
||||
'store-path-v1',
|
||||
'derivation-v3',
|
||||
'deriving-path-v1',
|
||||
]
|
||||
|
||||
schema_files = files()
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../../../../../src/libstore-tests/data/content-address
|
||||
@@ -0,0 +1,55 @@
|
||||
"$schema": "http://json-schema.org/draft-04/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json"
|
||||
title: Content Address
|
||||
description: |
|
||||
This schema describes the JSON representation of Nix's `ContentAddress` type, which conveys information about [content-addressing store objects](@docroot@/store/store-object/content-address.md).
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> For current methods of content addressing, this data type is a bit suspicious, because it is neither simply a content address of a file system object (the `method` is richer), nor simply a content address of a store object (the `hash` doesn't account for the references).
|
||||
> It should thus only be used in contexts where the references are also known / otherwise made tamper-resistant.
|
||||
|
||||
<!--
|
||||
TODO currently `ContentAddress` is used in both of these, and so same rationale applies, but actually in both cases the JSON is currently ad-hoc.
|
||||
That will be fixed, and as each is fixed, the example (along with a more precise link to the field in question) should be become part of the above note, so what is is saying is more clear.
|
||||
|
||||
> For example:
|
||||
|
||||
> - Fixed outputs of derivations are not allowed to have any references, so an empty reference set is statically known by assumption.
|
||||
|
||||
> - [Store object info](./store-object-info.md) includes the set of references along side the (optional) content address.
|
||||
|
||||
> This data type is thus safely used in both of these contexts.
|
||||
|
||||
-->
|
||||
|
||||
type: object
|
||||
properties:
|
||||
method:
|
||||
"$ref": "#/$defs/method"
|
||||
hash:
|
||||
title: Content Address
|
||||
description: |
|
||||
This would be the content-address itself.
|
||||
|
||||
For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/file-system-object/content-address.md), and not of the store object as a whole.
|
||||
In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods).
|
||||
"$ref": "./hash-v1.yaml"
|
||||
required:
|
||||
- method
|
||||
- hash
|
||||
additionalProperties: false
|
||||
"$defs":
|
||||
method:
|
||||
type: string
|
||||
enum: [flat, nar, text, git]
|
||||
title: Content-Addressing Method
|
||||
description: |
|
||||
A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
|
||||
|
||||
Valid method strings are:
|
||||
|
||||
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file)
|
||||
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
|
||||
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
|
||||
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
|
||||
@@ -1,5 +1,5 @@
|
||||
"$schema": http://json-schema.org/draft-04/schema#
|
||||
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json
|
||||
"$schema": "http://json-schema.org/draft-04/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json"
|
||||
title: Derivation
|
||||
description: |
|
||||
Experimental JSON representation of a Nix derivation (version 3).
|
||||
@@ -85,7 +85,7 @@ properties:
|
||||
> ]
|
||||
> ```
|
||||
items:
|
||||
type: string
|
||||
$ref: "store-path-v1.yaml"
|
||||
|
||||
inputDrvs:
|
||||
type: object
|
||||
@@ -103,6 +103,15 @@ properties:
|
||||
> ```
|
||||
>
|
||||
> specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
|
||||
patternProperties:
|
||||
"^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$":
|
||||
title: Store Path
|
||||
description: |
|
||||
A store path to a derivation, mapped to the outputs of that derivation.
|
||||
oneOf:
|
||||
- "$ref": "#/$defs/outputNames"
|
||||
- "$ref": "#/$defs/dynamicOutputs"
|
||||
additionalProperties: false
|
||||
|
||||
system:
|
||||
type: string
|
||||
@@ -148,25 +157,16 @@ properties:
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
$ref: "store-path-v1.yaml"
|
||||
title: Output path
|
||||
description: |
|
||||
The output path, if known in advance.
|
||||
|
||||
method:
|
||||
type: string
|
||||
title: Content addressing method
|
||||
enum: [flat, nar, text, git]
|
||||
"$ref": "./content-address-v1.yaml#/$defs/method"
|
||||
description: |
|
||||
For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
|
||||
|
||||
Valid method strings are:
|
||||
|
||||
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
|
||||
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
|
||||
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
|
||||
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
|
||||
|
||||
See the linked original definition for further details.
|
||||
hashAlgo:
|
||||
title: Hash algorithm
|
||||
"$ref": "./hash-v1.yaml#/$defs/algorithm"
|
||||
@@ -176,3 +176,28 @@ properties:
|
||||
title: Expected hash value
|
||||
description: |
|
||||
For fixed-output derivations, the expected content hash in base-16.
|
||||
|
||||
outputName:
|
||||
type: string
|
||||
title: Output name
|
||||
description: Name of the derivation output to depend on
|
||||
|
||||
outputNames:
|
||||
type: array
|
||||
title: Output Names
|
||||
description: Set of names of derivation outputs to depend on
|
||||
items:
|
||||
"$ref": "#/$defs/outputName"
|
||||
|
||||
dynamicOutputs:
|
||||
type: object
|
||||
title: Dynamic Outputs
|
||||
description: |
|
||||
**Experimental feature**: [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations)
|
||||
|
||||
This recursive data type allows for depending on outputs of outputs.
|
||||
properties:
|
||||
outputs:
|
||||
"$ref": "#/$defs/outputNames"
|
||||
dynamicOutputs:
|
||||
"$ref": "#/$defs/dynamicOutputs"
|
||||
@@ -0,0 +1 @@
|
||||
../../../../../../src/libstore-tests/data/derived-path
|
||||
@@ -0,0 +1,27 @@
|
||||
"$schema": "http://json-schema.org/draft-04/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json"
|
||||
title: Deriving Path
|
||||
description: |
|
||||
This schema describes the JSON representation of Nix's [Deriving Path](@docroot@/store/derivation/index.md#deriving-path).
|
||||
oneOf:
|
||||
- title: Constant
|
||||
description: |
|
||||
See [Constant](@docroot@/store/derivation/index.md#deriving-path-constant) deriving path.
|
||||
$ref: "store-path-v1.yaml"
|
||||
- title: Output
|
||||
description: |
|
||||
See [Output](@docroot@/store/derivation/index.md#deriving-path-output) deriving path.
|
||||
type: object
|
||||
properties:
|
||||
drvPath:
|
||||
"$ref": "#"
|
||||
description: |
|
||||
A deriving path to a [Derivation](@docroot@/store/derivation/index.md#store-derivation), whose output is being referred to.
|
||||
output:
|
||||
type: string
|
||||
description: |
|
||||
The name of an output produced by that derivation (e.g. "out", "doc", etc.).
|
||||
required:
|
||||
- drvPath
|
||||
- output
|
||||
additionalProperties: false
|
||||
@@ -1,5 +1,5 @@
|
||||
"$schema": http://json-schema.org/draft-04/schema#
|
||||
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json
|
||||
"$schema": "http://json-schema.org/draft-04/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json"
|
||||
title: Hash
|
||||
description: |
|
||||
A cryptographic hash value used throughout Nix for content addressing and integrity verification.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../../../../../src/libstore-tests/data/store-path
|
||||
@@ -0,0 +1,32 @@
|
||||
"$schema": "http://json-schema.org/draft-07/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-path-v1.json"
|
||||
title: Store Path
|
||||
description: |
|
||||
A [store path](@docroot@/store/store-path.md) identifying a store object.
|
||||
|
||||
This schema describes the JSON representation of store paths as used in various Nix JSON APIs.
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
> This JSON format is currently
|
||||
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
|
||||
> and subject to change.
|
||||
|
||||
## Format
|
||||
|
||||
Store paths in JSON are represented as strings containing just the hash and name portion, without the store directory prefix.
|
||||
|
||||
For example: `"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"`
|
||||
|
||||
(If the store dir is `/nix/store`, then this corresponds to the path `/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv`.)
|
||||
|
||||
## Structure
|
||||
|
||||
The format follows this pattern: `${digest}-${name}`
|
||||
|
||||
- **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
|
||||
- **name**: The package name and optional version/suffix information
|
||||
|
||||
type: string
|
||||
pattern: "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$"
|
||||
minLength: 34
|
||||
@@ -0,0 +1,15 @@
|
||||
{{#include store-path-v1-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### Simple store path
|
||||
|
||||
```json
|
||||
{{#include schema/store-path-v1/simple.json}}
|
||||
```
|
||||
|
||||
<!--
|
||||
## Raw Schema
|
||||
|
||||
[JSON Schema for Store Path v1](schema/store-path-v1.json)
|
||||
-->
|
||||
@@ -471,6 +471,27 @@
|
||||
}
|
||||
);
|
||||
|
||||
apps = forAllSystems (
|
||||
system:
|
||||
let
|
||||
pkgs = nixpkgsFor.${system}.native;
|
||||
opener = if pkgs.stdenv.isDarwin then "open" else "xdg-open";
|
||||
in
|
||||
{
|
||||
open-manual = {
|
||||
type = "app";
|
||||
program = "${pkgs.writeShellScript "open-nix-manual" ''
|
||||
manual_path="${self.packages.${system}.nix-manual}/share/doc/nix/manual/index.html"
|
||||
if ! ${opener} "$manual_path"; then
|
||||
echo "Failed to open manual with ${opener}. Manual is located at:"
|
||||
echo "$manual_path"
|
||||
fi
|
||||
''}";
|
||||
meta.description = "Open the Nix manual in your browser";
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
devShells =
|
||||
let
|
||||
makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; };
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# shellcheck disable=all
|
||||
#compdef nix
|
||||
# shellcheck disable=all
|
||||
|
||||
function _nix() {
|
||||
local ifs_bk="$IFS"
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../src/libstore-tests/data/content-address
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../src/libstore-tests/data/derived-path
|
||||
@@ -30,6 +30,21 @@ schemas = [
|
||||
'blake3-base64.json',
|
||||
],
|
||||
},
|
||||
{
|
||||
'stem' : 'content-address',
|
||||
'schema' : schema_dir / 'content-address-v1.yaml',
|
||||
'files' : [
|
||||
'text.json',
|
||||
'nar.json',
|
||||
],
|
||||
},
|
||||
{
|
||||
'stem' : 'store-path',
|
||||
'schema' : schema_dir / 'store-path-v1.yaml',
|
||||
'files' : [
|
||||
'simple.json',
|
||||
],
|
||||
},
|
||||
{
|
||||
'stem' : 'derivation',
|
||||
'schema' : schema_dir / 'derivation-v3.yaml',
|
||||
@@ -38,20 +53,28 @@ schemas = [
|
||||
'simple-derivation.json',
|
||||
],
|
||||
},
|
||||
# # Not sure how to make subschema work
|
||||
# {
|
||||
# 'stem': 'derivation',
|
||||
# 'schema': schema_dir / 'derivation-v3.yaml#output',
|
||||
# 'files' : [
|
||||
# 'output-caFixedFlat.json',
|
||||
# 'output-caFixedNAR.json',
|
||||
# 'output-caFixedText.json',
|
||||
# 'output-caFloating.json',
|
||||
# 'output-deferred.json',
|
||||
# 'output-impure.json',
|
||||
# 'output-inputAddressed.json',
|
||||
# ],
|
||||
# },
|
||||
{
|
||||
'stem' : 'derivation',
|
||||
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output',
|
||||
'files' : [
|
||||
'output-caFixedFlat.json',
|
||||
'output-caFixedNAR.json',
|
||||
'output-caFixedText.json',
|
||||
'output-caFloating.json',
|
||||
'output-deferred.json',
|
||||
'output-impure.json',
|
||||
'output-inputAddressed.json',
|
||||
],
|
||||
},
|
||||
{
|
||||
'stem' : 'deriving-path',
|
||||
'schema' : schema_dir / 'deriving-path-v1.yaml',
|
||||
'files' : [
|
||||
'single_opaque.json',
|
||||
'single_built.json',
|
||||
'single_built_built.json',
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# Validate each example against the schema
|
||||
@@ -64,8 +87,6 @@ foreach schema : schemas
|
||||
stem + '-schema-valid',
|
||||
jv,
|
||||
args : [
|
||||
'--map',
|
||||
'./hash-v1.yaml=' + schema_dir / 'hash-v1.yaml',
|
||||
'http://json-schema.org/draft-04/schema',
|
||||
schema_file,
|
||||
],
|
||||
|
||||
@@ -21,7 +21,10 @@ mkMesonDerivation (finalAttrs: {
|
||||
../../.version
|
||||
../../doc/manual/source/protocols/json/schema
|
||||
../../src/libutil-tests/data/hash
|
||||
../../src/libstore-tests/data/content-address
|
||||
../../src/libstore-tests/data/store-path
|
||||
../../src/libstore-tests/data/derivation
|
||||
../../src/libstore-tests/data/derived-path
|
||||
./.
|
||||
];
|
||||
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../src/libstore-tests/data/store-path
|
||||
@@ -10,7 +10,7 @@ using namespace testing;
|
||||
struct ValuePrintingTests : LibExprTest
|
||||
{
|
||||
template<class... A>
|
||||
void test(Value v, std::string_view expected, A... args)
|
||||
void test(Value & v, std::string_view expected, A... args)
|
||||
{
|
||||
std::stringstream out;
|
||||
v.print(state, out, args...);
|
||||
@@ -625,10 +625,11 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided)
|
||||
vThree.mkInt(3);
|
||||
|
||||
builder.insert(state.symbols.create("three"), &vThree);
|
||||
vAttrs.mkAttrs(builder.finish());
|
||||
Value vAttrs2;
|
||||
vAttrs2.mkAttrs(builder.finish());
|
||||
|
||||
test(
|
||||
vAttrs,
|
||||
vAttrs2,
|
||||
"{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }",
|
||||
PrintOptions{.ansiColors = true, .maxAttrs = 1});
|
||||
}
|
||||
|
||||
+11
-7
@@ -2050,10 +2050,10 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
|
||||
};
|
||||
|
||||
// List of returned strings. References to these Values must NOT be persisted.
|
||||
SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
|
||||
SmallTemporaryValueVector<conservativeStackReservation> values(es.size());
|
||||
Value * vTmpP = values.data();
|
||||
|
||||
for (auto & [i_pos, i] : *es) {
|
||||
for (auto & [i_pos, i] : es) {
|
||||
Value & vTmp = *vTmpP++;
|
||||
i->eval(state, env, vTmp);
|
||||
|
||||
@@ -2097,7 +2097,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
|
||||
.debugThrow();
|
||||
} else {
|
||||
if (s.empty())
|
||||
s.reserve(es->size());
|
||||
s.reserve(es.size());
|
||||
/* skip canonization of first path, which would only be not
|
||||
canonized in the first place if it's coming from a ./${foo} type
|
||||
path */
|
||||
@@ -3067,7 +3067,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path)
|
||||
return parseExprFromFile(path, staticBaseEnv);
|
||||
}
|
||||
|
||||
Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
|
||||
Expr * EvalState::parseExprFromFile(const SourcePath & path, const std::shared_ptr<StaticEnv> & staticEnv)
|
||||
{
|
||||
auto buffer = path.resolveSymlinks().readFile();
|
||||
// readFile hopefully have left some extra space for terminators
|
||||
@@ -3075,8 +3075,8 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<Sta
|
||||
return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
|
||||
}
|
||||
|
||||
Expr *
|
||||
EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
|
||||
Expr * EvalState::parseExprFromString(
|
||||
std::string s_, const SourcePath & basePath, const std::shared_ptr<StaticEnv> & staticEnv)
|
||||
{
|
||||
// NOTE this method (and parseStdin) must take care to *fully copy* their input
|
||||
// into their respective Pos::Origin until the parser stops overwriting its input
|
||||
@@ -3210,7 +3210,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
|
||||
}
|
||||
|
||||
Expr * EvalState::parse(
|
||||
char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
|
||||
char * text,
|
||||
size_t length,
|
||||
Pos::Origin origin,
|
||||
const SourcePath & basePath,
|
||||
const std::shared_ptr<StaticEnv> & staticEnv)
|
||||
{
|
||||
DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath
|
||||
DocCommentMap * docComments = &tmpDocComments;
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "nix/expr/symbol-table.hh"
|
||||
|
||||
#include <boost/container/static_vector.hpp>
|
||||
#include <boost/iterator/function_output_iterator.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
@@ -463,12 +464,48 @@ private:
|
||||
return bindings->baseLayer;
|
||||
}
|
||||
|
||||
/**
|
||||
* If the bindings gets "layered" on top of another we need to recalculate
|
||||
* the number of unique attributes in the chain.
|
||||
*
|
||||
* This is done by either iterating over the base "layer" and the newly added
|
||||
* attributes and counting duplicates. If the base "layer" is big this approach
|
||||
* is inefficient and we fall back to doing per-element binary search in the base
|
||||
* "layer".
|
||||
*/
|
||||
void finishSizeIfNecessary()
|
||||
{
|
||||
if (hasBaseLayer())
|
||||
/* NOTE: Do not use std::ranges::distance, since Bindings is a sized
|
||||
range, but we are calculating this size here. */
|
||||
bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end());
|
||||
if (!hasBaseLayer())
|
||||
return;
|
||||
|
||||
auto & base = *bindings->baseLayer;
|
||||
auto attrs = std::span(bindings->attrs, bindings->numAttrs);
|
||||
|
||||
Bindings::size_type duplicates = 0;
|
||||
|
||||
/* If the base bindings is smaller than the newly added attributes
|
||||
iterate using std::set_intersection to run in O(|base| + |attrs|) =
|
||||
O(|attrs|). Otherwise use an O(|attrs| * log(|base|)) per-attr binary
|
||||
search to check for duplicates. Note that if we are in this code path then
|
||||
|attrs| <= bindingsUpdateLayerRhsSizeThreshold, which 16 by default. We are
|
||||
optimizing for the case when a small attribute set gets "layered" on top of
|
||||
a much larger one. When attrsets are already small it's fine to do a linear
|
||||
scan, but we should avoid expensive iterations over large "base" attrsets. */
|
||||
if (attrs.size() > base.size()) {
|
||||
std::set_intersection(
|
||||
base.begin(),
|
||||
base.end(),
|
||||
attrs.begin(),
|
||||
attrs.end(),
|
||||
boost::make_function_output_iterator([&]([[maybe_unused]] auto && _) { ++duplicates; }));
|
||||
} else {
|
||||
for (const auto & attr : attrs) {
|
||||
if (base.get(attr.name))
|
||||
++duplicates;
|
||||
}
|
||||
}
|
||||
|
||||
bindings->numAttrsInChain = base.numAttrsInChain + attrs.size() - duplicates;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
@@ -191,7 +191,7 @@ std::ostream & operator<<(std::ostream & os, const ValueType t);
|
||||
|
||||
struct RegexCache;
|
||||
|
||||
std::shared_ptr<RegexCache> makeRegexCache();
|
||||
ref<RegexCache> makeRegexCache();
|
||||
|
||||
struct DebugTrace
|
||||
{
|
||||
@@ -372,6 +372,7 @@ public:
|
||||
|
||||
const fetchers::Settings & fetchSettings;
|
||||
const EvalSettings & settings;
|
||||
|
||||
SymbolTable symbols;
|
||||
PosTable positions;
|
||||
|
||||
@@ -418,7 +419,7 @@ public:
|
||||
|
||||
RootValue vImportedDrvToDerivation = nullptr;
|
||||
|
||||
ref<fetchers::InputCache> inputCache;
|
||||
const ref<fetchers::InputCache> inputCache;
|
||||
|
||||
/**
|
||||
* Debugger
|
||||
@@ -471,18 +472,18 @@ private:
|
||||
|
||||
/* Cache for calls to addToStore(); maps source paths to the store
|
||||
paths. */
|
||||
ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
|
||||
const ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
|
||||
|
||||
/**
|
||||
* A cache that maps paths to "resolved" paths for importing Nix
|
||||
* expressions, i.e. `/foo` to `/foo/default.nix`.
|
||||
*/
|
||||
ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
|
||||
const ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
|
||||
|
||||
/**
|
||||
* A cache from resolved paths to values.
|
||||
*/
|
||||
ref<boost::concurrent_flat_map<
|
||||
const ref<boost::concurrent_flat_map<
|
||||
SourcePath,
|
||||
Value *,
|
||||
std::hash<SourcePath>,
|
||||
@@ -504,7 +505,7 @@ private:
|
||||
/**
|
||||
* Cache used by prim_match().
|
||||
*/
|
||||
std::shared_ptr<RegexCache> regexCache;
|
||||
const ref<RegexCache> regexCache;
|
||||
|
||||
public:
|
||||
|
||||
@@ -592,12 +593,13 @@ public:
|
||||
* Parse a Nix expression from the specified file.
|
||||
*/
|
||||
Expr * parseExprFromFile(const SourcePath & path);
|
||||
Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
|
||||
Expr * parseExprFromFile(const SourcePath & path, const std::shared_ptr<StaticEnv> & staticEnv);
|
||||
|
||||
/**
|
||||
* Parse a Nix expression from the specified string.
|
||||
*/
|
||||
Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
|
||||
Expr *
|
||||
parseExprFromString(std::string s, const SourcePath & basePath, const std::shared_ptr<StaticEnv> & staticEnv);
|
||||
Expr * parseExprFromString(std::string s, const SourcePath & basePath);
|
||||
|
||||
Expr * parseStdin();
|
||||
@@ -766,7 +768,7 @@ public:
|
||||
|
||||
#if NIX_USE_BOEHMGC
|
||||
/** A GC root for the baseEnv reference. */
|
||||
std::shared_ptr<Env *> baseEnvP;
|
||||
const std::shared_ptr<Env *> baseEnvP;
|
||||
#endif
|
||||
|
||||
public:
|
||||
@@ -780,7 +782,7 @@ public:
|
||||
/**
|
||||
* The same, but used during parsing to resolve variables.
|
||||
*/
|
||||
std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
|
||||
const std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
|
||||
|
||||
/**
|
||||
* Internal primops not exposed to the user.
|
||||
@@ -862,7 +864,7 @@ private:
|
||||
size_t length,
|
||||
Pos::Origin origin,
|
||||
const SourcePath & basePath,
|
||||
std::shared_ptr<StaticEnv> & staticEnv);
|
||||
const std::shared_ptr<StaticEnv> & staticEnv);
|
||||
|
||||
/**
|
||||
* Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run
|
||||
|
||||
@@ -442,8 +442,14 @@ struct ExprAttrs : Expr
|
||||
|
||||
struct ExprList : Expr
|
||||
{
|
||||
std::vector<Expr *> elems;
|
||||
ExprList() {};
|
||||
std::span<Expr *> elems;
|
||||
|
||||
ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::vector<Expr *> exprs)
|
||||
: elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()})
|
||||
{
|
||||
std::ranges::copy(exprs, elems.begin());
|
||||
};
|
||||
|
||||
COMMON_METHODS
|
||||
Value * maybeThunk(EvalState & state, Env & env) override;
|
||||
|
||||
@@ -695,11 +701,11 @@ struct ExprConcatStrings : Expr
|
||||
{
|
||||
PosIdx pos;
|
||||
bool forceString;
|
||||
std::vector<std::pair<PosIdx, Expr *>> * es;
|
||||
ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
|
||||
std::vector<std::pair<PosIdx, Expr *>> es;
|
||||
ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> && es)
|
||||
: pos(pos)
|
||||
, forceString(forceString)
|
||||
, es(es) {};
|
||||
, es(std::move(es)) {};
|
||||
|
||||
PosIdx getPos() const override
|
||||
{
|
||||
|
||||
@@ -282,7 +282,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
|
||||
}
|
||||
|
||||
/* Strip spaces from each line. */
|
||||
auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
|
||||
std::vector<std::pair<PosIdx, Expr *>> es2{};
|
||||
atStartOfLine = true;
|
||||
size_t curDropped = 0;
|
||||
size_t n = es.size();
|
||||
@@ -290,7 +290,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
|
||||
const auto trimExpr = [&](Expr * e) {
|
||||
atStartOfLine = false;
|
||||
curDropped = 0;
|
||||
es2->emplace_back(i->first, e);
|
||||
es2.emplace_back(i->first, e);
|
||||
};
|
||||
const auto trimString = [&](const StringToken & t) {
|
||||
std::string s2;
|
||||
@@ -324,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
|
||||
|
||||
// Ignore empty strings for a minor optimisation and AST simplification
|
||||
if (s2 != "") {
|
||||
es2->emplace_back(i->first, new ExprString(alloc, s2));
|
||||
es2.emplace_back(i->first, new ExprString(alloc, s2));
|
||||
}
|
||||
};
|
||||
for (; i != es.end(); ++i, --n) {
|
||||
@@ -333,19 +333,17 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
|
||||
|
||||
// If there is nothing at all, return the empty string directly.
|
||||
// This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
|
||||
if (es2->size() == 0) {
|
||||
if (es2.size() == 0) {
|
||||
auto * const result = new ExprString("");
|
||||
delete es2;
|
||||
return result;
|
||||
}
|
||||
|
||||
/* If this is a single string, then don't do a concatenation. */
|
||||
if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
|
||||
auto * const result = (*es2)[0].second;
|
||||
delete es2;
|
||||
if (es2.size() == 1 && dynamic_cast<ExprString *>((es2)[0].second)) {
|
||||
auto * const result = (es2)[0].second;
|
||||
return result;
|
||||
}
|
||||
return new ExprConcatStrings(pos, true, es2);
|
||||
return new ExprConcatStrings(pos, true, std::move(es2));
|
||||
}
|
||||
|
||||
inline PosIdx LexerState::at(const ParserLocation & loc)
|
||||
|
||||
+17
-17
@@ -142,11 +142,11 @@ or { return OR_KW; }
|
||||
return PIPE_INTO;
|
||||
}
|
||||
|
||||
{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; }
|
||||
{ID} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return ID; }
|
||||
{INT} { errno = 0;
|
||||
std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
|
||||
if (numMay.has_value()) {
|
||||
yylval->n = NixInt{*numMay};
|
||||
yylval->emplace<NixInt>(*numMay);
|
||||
} else {
|
||||
throw ParseError(ErrorInfo{
|
||||
.msg = HintFmt("invalid integer '%1%'", yytext),
|
||||
@@ -156,7 +156,7 @@ or { return OR_KW; }
|
||||
return INT_LIT;
|
||||
}
|
||||
{FLOAT} { errno = 0;
|
||||
yylval->nf = strtod(yytext, 0);
|
||||
yylval->emplace<NixFloat>(strtod(yytext, 0));
|
||||
if (errno != 0)
|
||||
throw ParseError(ErrorInfo{
|
||||
.msg = HintFmt("invalid float '%1%'", yytext),
|
||||
@@ -183,7 +183,7 @@ or { return OR_KW; }
|
||||
/* It is impossible to match strings ending with '$' with one
|
||||
regex because trailing contexts are only valid at the end
|
||||
of a rule. (A sane but undocumented limitation.) */
|
||||
yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; });
|
||||
yylval->emplace<StringToken>(unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; }));
|
||||
return STR;
|
||||
}
|
||||
<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
|
||||
@@ -198,27 +198,27 @@ or { return OR_KW; }
|
||||
|
||||
\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
|
||||
<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
|
||||
yylval->str = {yytext, (size_t) yyleng, true};
|
||||
forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; });
|
||||
yylval->emplace<StringToken>(yytext, (size_t) yyleng, true);
|
||||
forceNoNullByte(yylval->as<StringToken>(), [&]() { return state->positions[CUR_POS]; });
|
||||
return IND_STR;
|
||||
}
|
||||
<IND_STRING>\'\'\$ |
|
||||
<IND_STRING>\$ {
|
||||
yylval->str = {"$", 1};
|
||||
yylval->emplace<StringToken>("$", 1);
|
||||
return IND_STR;
|
||||
}
|
||||
<IND_STRING>\'\'\' {
|
||||
yylval->str = {"''", 2};
|
||||
yylval->emplace<StringToken>("''", 2);
|
||||
return IND_STR;
|
||||
}
|
||||
<IND_STRING>\'\'\\{ANY} {
|
||||
yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; });
|
||||
yylval->emplace<StringToken>(unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; }));
|
||||
return IND_STR;
|
||||
}
|
||||
<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
|
||||
<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
|
||||
<IND_STRING>\' {
|
||||
yylval->str = {"'", 1};
|
||||
yylval->emplace<StringToken>("'", 1);
|
||||
return IND_STR;
|
||||
}
|
||||
|
||||
@@ -232,14 +232,14 @@ or { return OR_KW; }
|
||||
<PATH_START>{PATH_SEG} {
|
||||
POP_STATE();
|
||||
PUSH_STATE(INPATH_SLASH);
|
||||
yylval->path = {yytext, (size_t) yyleng};
|
||||
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
|
||||
return PATH;
|
||||
}
|
||||
|
||||
<PATH_START>{HPATH_START} {
|
||||
POP_STATE();
|
||||
PUSH_STATE(INPATH_SLASH);
|
||||
yylval->path = {yytext, (size_t) yyleng};
|
||||
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
|
||||
return HPATH;
|
||||
}
|
||||
|
||||
@@ -248,7 +248,7 @@ or { return OR_KW; }
|
||||
PUSH_STATE(INPATH_SLASH);
|
||||
else
|
||||
PUSH_STATE(INPATH);
|
||||
yylval->path = {yytext, (size_t) yyleng};
|
||||
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
|
||||
return PATH;
|
||||
}
|
||||
{HPATH} {
|
||||
@@ -256,7 +256,7 @@ or { return OR_KW; }
|
||||
PUSH_STATE(INPATH_SLASH);
|
||||
else
|
||||
PUSH_STATE(INPATH);
|
||||
yylval->path = {yytext, (size_t) yyleng};
|
||||
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
|
||||
return HPATH;
|
||||
}
|
||||
|
||||
@@ -272,7 +272,7 @@ or { return OR_KW; }
|
||||
PUSH_STATE(INPATH_SLASH);
|
||||
else
|
||||
PUSH_STATE(INPATH);
|
||||
yylval->str = {yytext, (size_t) yyleng};
|
||||
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
|
||||
return STR;
|
||||
}
|
||||
<INPATH>{ANY} |
|
||||
@@ -294,8 +294,8 @@ or { return OR_KW; }
|
||||
});
|
||||
}
|
||||
|
||||
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
|
||||
{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
|
||||
{SPATH} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return SPATH; }
|
||||
{URI} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return URI; }
|
||||
|
||||
%{
|
||||
// Doc comment rule
|
||||
|
||||
+47
-2
@@ -183,17 +183,62 @@ subdir('primops')
|
||||
subdir('nix-meson-build-support/export-all-symbols')
|
||||
subdir('nix-meson-build-support/windows-version')
|
||||
|
||||
# Turns out that Bison/Flex are particularly sensitive to compilers
|
||||
# failing to inline functions. For that reason we crank up the inlining
|
||||
# threshold manually for optimized builds. Yes, this can be considered 'ricing'
|
||||
# the compiler, but it does pay off.
|
||||
#
|
||||
# NOTE: missed inlining can be spotted (for Clang) using -Rpass-missed=inline
|
||||
# and -fdump-ipa-inline-missed (for GCC).
|
||||
parser_library_cpp_args = []
|
||||
|
||||
if not get_option('debug')
|
||||
if cxx.get_id() == 'clang'
|
||||
# The default as of LLVM 21 is 225:
|
||||
# llc --help-hidden | grep inline-threshold
|
||||
parser_library_cpp_args += [
|
||||
'-mllvm',
|
||||
'-inline-threshold=5000',
|
||||
]
|
||||
elif cxx.get_id() == 'gcc'
|
||||
parser_library_cpp_args += [
|
||||
'--param=max-inline-insns-single=1000',
|
||||
'--param=max-inline-insns-auto=1000',
|
||||
'--param=inline-unit-growth=400',
|
||||
]
|
||||
endif
|
||||
endif
|
||||
|
||||
# Working around https://github.com/mesonbuild/meson/issues/1367.
|
||||
parser_library = static_library(
|
||||
'nixexpr-parser',
|
||||
parser_tab,
|
||||
lexer_tab,
|
||||
cpp_args : parser_library_cpp_args,
|
||||
dependencies : deps_public + deps_private + deps_other,
|
||||
include_directories : include_dirs,
|
||||
# 1. Stdlib and regular assertions regress parser performance significantly, so build without
|
||||
# them for this one library when building in a release configuration.
|
||||
# 2. Disable LTO for GCC because then inlining flags won't apply, since LTO in GCC is done
|
||||
# by plonking down GIMPLE in the archive.
|
||||
override_options : [
|
||||
'b_ndebug=@0@'.format(not get_option('debug')),
|
||||
'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'),
|
||||
],
|
||||
)
|
||||
|
||||
this_library = library(
|
||||
'nixexpr',
|
||||
sources,
|
||||
config_priv_h,
|
||||
parser_tab,
|
||||
lexer_tab,
|
||||
parser_tab[1],
|
||||
lexer_tab[1],
|
||||
generated_headers,
|
||||
soversion : nix_soversion,
|
||||
dependencies : deps_public + deps_private + deps_other,
|
||||
include_directories : include_dirs,
|
||||
link_args : linker_export_flags,
|
||||
link_whole : [ parser_library ],
|
||||
prelink : true, # For C++ static initializers
|
||||
install : true,
|
||||
cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [],
|
||||
|
||||
@@ -246,7 +246,7 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co
|
||||
{
|
||||
bool first = true;
|
||||
str << "(";
|
||||
for (auto & i : *es) {
|
||||
for (auto & i : es) {
|
||||
if (first)
|
||||
first = false;
|
||||
else
|
||||
@@ -564,7 +564,7 @@ void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr<const Sta
|
||||
if (es.debugRepl)
|
||||
es.exprEnvs.insert(std::make_pair(this, env));
|
||||
|
||||
for (auto & i : *this->es)
|
||||
for (auto & i : this->es)
|
||||
i.second->bindVars(es, env);
|
||||
}
|
||||
|
||||
|
||||
+74
-98
@@ -14,6 +14,10 @@
|
||||
|
||||
%code requires {
|
||||
|
||||
// bison adds a bunch of switch statements with default:
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wswitch-enum"
|
||||
|
||||
#ifndef BISON_HEADER
|
||||
#define BISON_HEADER
|
||||
|
||||
@@ -120,46 +124,28 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
|
||||
|
||||
%}
|
||||
|
||||
%union {
|
||||
// !!! We're probably leaking stuff here.
|
||||
nix::Expr * e;
|
||||
nix::ExprList * list;
|
||||
nix::ExprAttrs * attrs;
|
||||
nix::Formals * formals;
|
||||
nix::Formal * formal;
|
||||
nix::NixInt n;
|
||||
nix::NixFloat nf;
|
||||
nix::StringToken id; // !!! -> Symbol
|
||||
nix::StringToken path;
|
||||
nix::StringToken uri;
|
||||
nix::StringToken str;
|
||||
std::vector<nix::AttrName> * attrNames;
|
||||
std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
|
||||
std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
|
||||
std::variant<nix::Expr *, std::string_view> * to_be_string;
|
||||
std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
|
||||
}
|
||||
%define api.value.type variant
|
||||
|
||||
%type <e> start expr expr_function expr_if expr_op
|
||||
%type <e> expr_select expr_simple expr_app
|
||||
%type <e> expr_pipe_from expr_pipe_into
|
||||
%type <list> expr_list
|
||||
%type <attrs> binds binds1
|
||||
%type <formals> formals formal_set
|
||||
%type <formal> formal
|
||||
%type <attrNames> attrpath
|
||||
%type <inheritAttrs> attrs
|
||||
%type <string_parts> string_parts_interpolated
|
||||
%type <ind_string_parts> ind_string_parts
|
||||
%type <e> path_start
|
||||
%type <to_be_string> string_parts string_attr
|
||||
%type <id> attr
|
||||
%token <id> ID
|
||||
%token <str> STR IND_STR
|
||||
%token <n> INT_LIT
|
||||
%token <nf> FLOAT_LIT
|
||||
%token <path> PATH HPATH SPATH PATH_END
|
||||
%token <uri> URI
|
||||
%type <nix::Expr *> start expr expr_function expr_if expr_op
|
||||
%type <nix::Expr *> expr_select expr_simple expr_app
|
||||
%type <nix::Expr *> expr_pipe_from expr_pipe_into
|
||||
%type <std::vector<Expr *>> list
|
||||
%type <nix::ExprAttrs *> binds binds1
|
||||
%type <nix::Formals *> formals formal_set
|
||||
%type <nix::Formal> formal
|
||||
%type <std::vector<nix::AttrName>> attrpath
|
||||
%type <std::vector<std::pair<nix::AttrName, nix::PosIdx>>> attrs
|
||||
%type <std::vector<std::pair<nix::PosIdx, nix::Expr *>>> string_parts_interpolated
|
||||
%type <std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>>> ind_string_parts
|
||||
%type <nix::Expr *> path_start
|
||||
%type <std::variant<nix::Expr *, std::string_view>> string_parts string_attr
|
||||
%type <nix::StringToken> attr
|
||||
%token <nix::StringToken> ID
|
||||
%token <nix::StringToken> STR IND_STR
|
||||
%token <nix::NixInt> INT_LIT
|
||||
%token <nix::NixFloat> FLOAT_LIT
|
||||
%token <nix::StringToken> PATH HPATH SPATH PATH_END
|
||||
%token <nix::StringToken> URI
|
||||
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
|
||||
%token PIPE_FROM PIPE_INTO /* <| and |> */
|
||||
%token DOLLAR_CURLY /* == ${ */
|
||||
@@ -261,9 +247,9 @@ expr_op
|
||||
| expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
|
||||
| expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
|
||||
| expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
|
||||
| expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move(*$3)); delete $3; }
|
||||
| expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move($3)); }
|
||||
| expr_op '+' expr_op
|
||||
{ $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
|
||||
{ $$ = new ExprConcatStrings(state->at(@2), false, {{state->at(@1), $1}, {state->at(@3), $3}}); }
|
||||
| expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
|
||||
| expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
|
||||
| expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
|
||||
@@ -282,9 +268,9 @@ expr_app
|
||||
|
||||
expr_select
|
||||
: expr_simple '.' attrpath
|
||||
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
|
||||
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move($3), nullptr); }
|
||||
| expr_simple '.' attrpath OR_KW expr_select
|
||||
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
|
||||
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move($3), $5); $5->warnIfCursedOr(state->symbols, state->positions); }
|
||||
| /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
|
||||
allow stuff like ‘map or [...]’. This production is problematic (see
|
||||
https://github.com/NixOS/nix/issues/11118) and will be refactored in the
|
||||
@@ -311,17 +297,15 @@ expr_simple
|
||||
std::visit(overloaded{
|
||||
[&](std::string_view str) { $$ = new ExprString(state->alloc, str); },
|
||||
[&](Expr * expr) { $$ = expr; }},
|
||||
*$2);
|
||||
delete $2;
|
||||
$2);
|
||||
}
|
||||
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
|
||||
$$ = state->stripIndentation(CUR_POS, std::move(*$2));
|
||||
delete $2;
|
||||
$$ = state->stripIndentation(CUR_POS, std::move($2));
|
||||
}
|
||||
| path_start PATH_END
|
||||
| path_start string_parts_interpolated PATH_END {
|
||||
$2->insert($2->begin(), {state->at(@1), $1});
|
||||
$$ = new ExprConcatStrings(CUR_POS, false, $2);
|
||||
$2.insert($2.begin(), {state->at(@1), $1});
|
||||
$$ = new ExprConcatStrings(CUR_POS, false, std::move($2));
|
||||
}
|
||||
| SPATH {
|
||||
std::string_view path($1.p + 1, $1.l - 2);
|
||||
@@ -350,24 +334,23 @@ expr_simple
|
||||
{ $2->pos = CUR_POS; $$ = $2; }
|
||||
| '{' '}'
|
||||
{ $$ = new ExprAttrs(CUR_POS); }
|
||||
| '[' expr_list ']' { $$ = $2; }
|
||||
| '[' list ']' { $$ = new ExprList(state->alloc, std::move($2)); }
|
||||
;
|
||||
|
||||
string_parts
|
||||
: STR { $$ = new std::variant<Expr *, std::string_view>($1); }
|
||||
| string_parts_interpolated { $$ = new std::variant<Expr *, std::string_view>(new ExprConcatStrings(CUR_POS, true, $1)); }
|
||||
| { $$ = new std::variant<Expr *, std::string_view>(std::string_view()); }
|
||||
: STR { $$ = $1; }
|
||||
| string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, std::move($1)); }
|
||||
| { $$ = std::string_view(); }
|
||||
;
|
||||
|
||||
string_parts_interpolated
|
||||
: string_parts_interpolated STR
|
||||
{ $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
|
||||
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
|
||||
| DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
|
||||
{ $$ = std::move($1); $$.emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
|
||||
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); }
|
||||
| DOLLAR_CURLY expr '}' { $$.emplace_back(state->at(@1), $2); }
|
||||
| STR DOLLAR_CURLY expr '}' {
|
||||
$$ = new std::vector<std::pair<PosIdx, Expr *>>;
|
||||
$$->emplace_back(state->at(@1), new ExprString(state->alloc, $1));
|
||||
$$->emplace_back(state->at(@2), $3);
|
||||
$$.emplace_back(state->at(@1), new ExprString(state->alloc, $1));
|
||||
$$.emplace_back(state->at(@2), $3);
|
||||
}
|
||||
;
|
||||
|
||||
@@ -408,9 +391,9 @@ path_start
|
||||
;
|
||||
|
||||
ind_string_parts
|
||||
: ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
|
||||
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
|
||||
| { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
|
||||
: ind_string_parts IND_STR { $$ = std::move($1); $$.emplace_back(state->at(@2), $2); }
|
||||
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); }
|
||||
| { }
|
||||
;
|
||||
|
||||
binds
|
||||
@@ -421,19 +404,17 @@ binds
|
||||
binds1
|
||||
: binds1[accum] attrpath '=' expr ';'
|
||||
{ $$ = $accum;
|
||||
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
|
||||
delete $attrpath;
|
||||
state->addAttr($$, std::move($attrpath), @attrpath, $expr, @expr);
|
||||
}
|
||||
| binds[accum] INHERIT attrs ';'
|
||||
{ $$ = $accum;
|
||||
for (auto & [i, iPos] : *$attrs) {
|
||||
for (auto & [i, iPos] : $attrs) {
|
||||
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
|
||||
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
|
||||
$accum->attrs.emplace(
|
||||
i.symbol,
|
||||
ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
|
||||
}
|
||||
delete $attrs;
|
||||
}
|
||||
| binds[accum] INHERIT '(' expr ')' attrs ';'
|
||||
{ $$ = $accum;
|
||||
@@ -441,7 +422,7 @@ binds1
|
||||
$accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
|
||||
$accum->inheritFromExprs->push_back($expr);
|
||||
auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
|
||||
for (auto & [i, iPos] : *$attrs) {
|
||||
for (auto & [i, iPos] : $attrs) {
|
||||
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
|
||||
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
|
||||
$accum->attrs.emplace(
|
||||
@@ -451,51 +432,45 @@ binds1
|
||||
iPos,
|
||||
ExprAttrs::AttrDef::Kind::InheritedFrom));
|
||||
}
|
||||
delete $attrs;
|
||||
}
|
||||
| attrpath '=' expr ';'
|
||||
{ $$ = new ExprAttrs;
|
||||
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
|
||||
delete $attrpath;
|
||||
state->addAttr($$, std::move($attrpath), @attrpath, $expr, @expr);
|
||||
}
|
||||
;
|
||||
|
||||
attrs
|
||||
: attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
|
||||
: attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); }
|
||||
| attrs string_attr
|
||||
{ $$ = $1;
|
||||
{ $$ = std::move($1);
|
||||
std::visit(overloaded {
|
||||
[&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); },
|
||||
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); },
|
||||
[&](Expr * expr) {
|
||||
throw ParseError({
|
||||
.msg = HintFmt("dynamic attributes not allowed in inherit"),
|
||||
.pos = state->positions[state->at(@2)]
|
||||
});
|
||||
}
|
||||
}, *$2);
|
||||
delete $2;
|
||||
}, $2);
|
||||
}
|
||||
| { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
|
||||
| { }
|
||||
;
|
||||
|
||||
attrpath
|
||||
: attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
|
||||
: attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); }
|
||||
| attrpath '.' string_attr
|
||||
{ $$ = $1;
|
||||
{ $$ = std::move($1);
|
||||
std::visit(overloaded {
|
||||
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
|
||||
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
|
||||
}, *$3);
|
||||
delete $3;
|
||||
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
|
||||
[&](Expr * expr) { $$.emplace_back(expr); }
|
||||
}, std::move($3));
|
||||
}
|
||||
| attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
|
||||
| attr { $$.emplace_back(state->symbols.create($1)); }
|
||||
| string_attr
|
||||
{ $$ = new std::vector<AttrName>;
|
||||
std::visit(overloaded {
|
||||
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
|
||||
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
|
||||
}, *$1);
|
||||
delete $1;
|
||||
{ std::visit(overloaded {
|
||||
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
|
||||
[&](Expr * expr) { $$.emplace_back(expr); }
|
||||
}, std::move($1));
|
||||
}
|
||||
;
|
||||
|
||||
@@ -505,13 +480,13 @@ attr
|
||||
;
|
||||
|
||||
string_attr
|
||||
: '"' string_parts '"' { $$ = $2; }
|
||||
| DOLLAR_CURLY expr '}' { $$ = new std::variant<Expr *, std::string_view>($2); }
|
||||
: '"' string_parts '"' { $$ = std::move($2); }
|
||||
| DOLLAR_CURLY expr '}' { $$ = $2; }
|
||||
;
|
||||
|
||||
expr_list
|
||||
: expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
|
||||
| { $$ = new ExprList; }
|
||||
list
|
||||
: list expr_select { $$ = std::move($1); $$.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
|
||||
| { }
|
||||
;
|
||||
|
||||
formal_set
|
||||
@@ -524,14 +499,14 @@ formal_set
|
||||
|
||||
formals
|
||||
: formals[accum] ',' formal
|
||||
{ $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; }
|
||||
{ $$ = $accum; $$->formals.emplace_back(std::move($formal)); }
|
||||
| formal
|
||||
{ $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; }
|
||||
{ $$ = new Formals; $$->formals.emplace_back(std::move($formal)); }
|
||||
;
|
||||
|
||||
formal
|
||||
: ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
|
||||
| ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
|
||||
: ID { $$ = Formal{CUR_POS, state->symbols.create($1), 0}; }
|
||||
| ID '?' expr { $$ = Formal{CUR_POS, state->symbols.create($1), $3}; }
|
||||
;
|
||||
|
||||
%%
|
||||
@@ -582,3 +557,4 @@ Expr * parseExprFromBuf(
|
||||
|
||||
|
||||
}
|
||||
#pragma GCC diagnostic pop // end ignored "-Wswitch-enum"
|
||||
@@ -1374,7 +1374,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
|
||||
pos,
|
||||
"while evaluating the `__structuredAttrs` "
|
||||
"attribute passed to builtins.derivationStrict"))
|
||||
jsonObject = StructuredAttrs{.structuredAttrs = json::object()};
|
||||
jsonObject = StructuredAttrs{};
|
||||
|
||||
/* Check whether null attributes should be ignored. */
|
||||
bool ignoreNulls = false;
|
||||
@@ -4611,9 +4611,9 @@ struct RegexCache
|
||||
}
|
||||
};
|
||||
|
||||
std::shared_ptr<RegexCache> makeRegexCache()
|
||||
ref<RegexCache> makeRegexCache()
|
||||
{
|
||||
return std::make_shared<RegexCache>();
|
||||
return make_ref<RegexCache>();
|
||||
}
|
||||
|
||||
void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
|
||||
@@ -64,6 +64,8 @@ static void runFetchClosureWithRewrite(
|
||||
.pos = state.positions[pos]});
|
||||
}
|
||||
|
||||
state.allowClosure(toPath);
|
||||
|
||||
state.mkStorePathString(toPath, v);
|
||||
}
|
||||
|
||||
@@ -91,6 +93,8 @@ static void runFetchClosureWithContentAddressedPath(
|
||||
.pos = state.positions[pos]});
|
||||
}
|
||||
|
||||
state.allowClosure(fromPath);
|
||||
|
||||
state.mkStorePathString(fromPath, v);
|
||||
}
|
||||
|
||||
@@ -115,6 +119,8 @@ static void runFetchClosureWithInputAddressedPath(
|
||||
.pos = state.positions[pos]});
|
||||
}
|
||||
|
||||
state.allowClosure(fromPath);
|
||||
|
||||
state.mkStorePathString(fromPath, v);
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "nix/util/json-utils.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/util/url.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@@ -65,6 +66,12 @@ Input Input::fromURL(const Settings & settings, const ParsedURL & url, bool requ
|
||||
}
|
||||
}
|
||||
|
||||
// Provide a helpful hint when user tries file+git instead of git+file
|
||||
auto parsedScheme = parseUrlScheme(url.scheme);
|
||||
if (parsedScheme.application == "file" && parsedScheme.transport == "git") {
|
||||
throw Error("input '%s' is unsupported; did you mean 'git+file' instead of 'file+git'?", url);
|
||||
}
|
||||
|
||||
throw Error("input '%s' is unsupported", url);
|
||||
}
|
||||
|
||||
@@ -512,10 +519,11 @@ using namespace nix;
|
||||
fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json & json)
|
||||
{
|
||||
fetchers::PublicKey res = {};
|
||||
if (auto type = optionalValueAt(json, "type"))
|
||||
auto & obj = getObject(json);
|
||||
if (auto * type = optionalValueAt(obj, "type"))
|
||||
res.type = getString(*type);
|
||||
|
||||
res.key = getString(valueAt(json, "key"));
|
||||
res.key = getString(valueAt(obj, "key"));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
|
||||
auto url = api.endpoint + "/objects/batch";
|
||||
const auto & authHeader = api.authHeader;
|
||||
FileTransferRequest request(parseURL(url));
|
||||
request.post = true;
|
||||
request.method = HttpMethod::POST;
|
||||
Headers headers;
|
||||
if (authHeader.has_value())
|
||||
headers.push_back({"Authorization", *authHeader});
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "nix/util/users.hh"
|
||||
#include "nix/util/fs-sink.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include <git2/attr.h>
|
||||
#include <git2/blob.h>
|
||||
@@ -530,12 +531,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
auto act = (Activity *) payload;
|
||||
act->result(
|
||||
resFetchStatus,
|
||||
fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
|
||||
fmt("%d/%d objects received, %d/%d deltas indexed, %s",
|
||||
stats->received_objects,
|
||||
stats->total_objects,
|
||||
stats->indexed_deltas,
|
||||
stats->total_deltas,
|
||||
stats->received_bytes / (1024.0 * 1024.0)));
|
||||
renderSize(stats->received_bytes)));
|
||||
return getInterrupted() ? -1 : 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -89,8 +89,6 @@ extern volatile ::sig_atomic_t blockInt;
|
||||
|
||||
/* GC helpers. */
|
||||
|
||||
std::string showBytes(uint64_t bytes);
|
||||
|
||||
struct GCResults;
|
||||
|
||||
struct PrintFreed
|
||||
|
||||
+31
-34
@@ -6,6 +6,7 @@
|
||||
#include "nix/main/loggers.hh"
|
||||
#include "nix/main/progress-bar.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <exception>
|
||||
@@ -64,18 +65,19 @@ void printMissing(ref<Store> store, const MissingPaths & missing, Verbosity lvl)
|
||||
}
|
||||
|
||||
if (!missing.willSubstitute.empty()) {
|
||||
const float downloadSizeMiB = missing.downloadSize / (1024.f * 1024.f);
|
||||
const float narSizeMiB = missing.narSize / (1024.f * 1024.f);
|
||||
if (missing.willSubstitute.size() == 1) {
|
||||
printMsg(
|
||||
lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", downloadSizeMiB, narSizeMiB);
|
||||
lvl,
|
||||
"this path will be fetched (%s download, %s unpacked):",
|
||||
renderSize(missing.downloadSize),
|
||||
renderSize(missing.narSize));
|
||||
} else {
|
||||
printMsg(
|
||||
lvl,
|
||||
"these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
|
||||
"these %d paths will be fetched (%s download, %s unpacked):",
|
||||
missing.willSubstitute.size(),
|
||||
downloadSizeMiB,
|
||||
narSizeMiB);
|
||||
renderSize(missing.downloadSize),
|
||||
renderSize(missing.narSize));
|
||||
}
|
||||
std::vector<const StorePath *> willSubstituteSorted = {};
|
||||
std::for_each(missing.willSubstitute.begin(), missing.willSubstitute.end(), [&](const StorePath & p) {
|
||||
@@ -320,34 +322,29 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
|
||||
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
|
||||
try {
|
||||
try {
|
||||
try {
|
||||
fun();
|
||||
} catch (...) {
|
||||
/* Subtle: we have to make sure that any `interrupted'
|
||||
condition is discharged before we reach printMsg()
|
||||
below, since otherwise it will throw an (uncaught)
|
||||
exception. */
|
||||
setInterruptThrown();
|
||||
throw;
|
||||
}
|
||||
} catch (Exit & e) {
|
||||
return e.status;
|
||||
} catch (UsageError & e) {
|
||||
logError(e.info());
|
||||
printError("Try '%1% --help' for more information.", programName);
|
||||
return 1;
|
||||
} catch (BaseError & e) {
|
||||
logError(e.info());
|
||||
return e.info().status;
|
||||
} catch (std::bad_alloc & e) {
|
||||
printError(error + "out of memory");
|
||||
return 1;
|
||||
} catch (std::exception & e) {
|
||||
printError(error + e.what());
|
||||
return 1;
|
||||
fun();
|
||||
} catch (...) {
|
||||
/* Subtle: we have to make sure that any `interrupted'
|
||||
condition is discharged before we reach printMsg()
|
||||
below, since otherwise it will throw an (uncaught)
|
||||
exception. */
|
||||
setInterruptThrown();
|
||||
throw;
|
||||
}
|
||||
} catch (...) {
|
||||
/* In case logger also throws just give up. */
|
||||
} catch (Exit & e) {
|
||||
return e.status;
|
||||
} catch (UsageError & e) {
|
||||
logError(e.info());
|
||||
printError("Try '%1% --help' for more information.", programName);
|
||||
return 1;
|
||||
} catch (BaseError & e) {
|
||||
logError(e.info());
|
||||
return e.info().status;
|
||||
} catch (std::bad_alloc & e) {
|
||||
printError(error + "out of memory");
|
||||
return 1;
|
||||
} catch (std::exception & e) {
|
||||
printError(error + e.what());
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -411,7 +408,7 @@ RunPager::~RunPager()
|
||||
PrintFreed::~PrintFreed()
|
||||
{
|
||||
if (show)
|
||||
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), showBytes(results.bytesFreed));
|
||||
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -126,6 +126,36 @@ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const c
|
||||
NIXC_CATCH_ERRS_NULL
|
||||
}
|
||||
|
||||
nix_err nix_store_get_fs_closure(
|
||||
nix_c_context * context,
|
||||
Store * store,
|
||||
const StorePath * store_path,
|
||||
bool flip_direction,
|
||||
bool include_outputs,
|
||||
bool include_derivers,
|
||||
void * userdata,
|
||||
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path))
|
||||
{
|
||||
if (context)
|
||||
context->last_err_code = NIX_OK;
|
||||
try {
|
||||
const auto nixStore = store->ptr;
|
||||
|
||||
nix::StorePathSet set;
|
||||
nixStore->computeFSClosure(store_path->path, set, flip_direction, include_outputs, include_derivers);
|
||||
|
||||
if (callback) {
|
||||
for (const auto & path : set) {
|
||||
const StorePath tmp{path};
|
||||
callback(context, userdata, &tmp);
|
||||
if (context && context->last_err_code != NIX_OK)
|
||||
return context->last_err_code;
|
||||
}
|
||||
}
|
||||
}
|
||||
NIXC_CATCH_ERRS
|
||||
}
|
||||
|
||||
nix_err nix_store_realise(
|
||||
nix_c_context * context,
|
||||
Store * store,
|
||||
@@ -143,6 +173,14 @@ nix_err nix_store_realise(
|
||||
const auto nixStore = store->ptr;
|
||||
auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore);
|
||||
|
||||
assert(results.size() == 1);
|
||||
|
||||
// Check if any builds failed
|
||||
for (auto & result : results) {
|
||||
if (auto * failureP = result.tryGetFailure())
|
||||
failureP->rethrow();
|
||||
}
|
||||
|
||||
if (callback) {
|
||||
for (const auto & result : results) {
|
||||
if (auto * success = result.tryGetSuccess()) {
|
||||
|
||||
@@ -186,6 +186,8 @@ nix_err nix_store_real_path(
|
||||
* @param[in] path Path to build
|
||||
* @param[in] userdata data to pass to every callback invocation
|
||||
* @param[in] callback called for every realised output
|
||||
* @return NIX_OK if the build succeeded, or an error code if the build/scheduling/outputs/copying/etc failed.
|
||||
* On error, the callback is never invoked and error information is stored in context.
|
||||
*/
|
||||
nix_err nix_store_realise(
|
||||
nix_c_context * context,
|
||||
@@ -245,6 +247,35 @@ void nix_derivation_free(nix_derivation * drv);
|
||||
*/
|
||||
nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path);
|
||||
|
||||
/**
|
||||
* @brief Gets the closure of a specific store path
|
||||
*
|
||||
* @note The callback borrows each StorePath only for the duration of the call.
|
||||
*
|
||||
* @param[out] context Optional, stores error information
|
||||
* @param[in] store nix store reference
|
||||
* @param[in] store_path The path to compute from
|
||||
* @param[in] flip_direction If false, compute the forward closure (paths referenced by any store path in the closure).
|
||||
* If true, compute the backward closure (paths that reference any store path in the closure).
|
||||
* @param[in] include_outputs If flip_direction is false: for any derivation in the closure, include its outputs.
|
||||
* If flip_direction is true: for any output in the closure, include derivations that produce
|
||||
* it.
|
||||
* @param[in] include_derivers If flip_direction is false: for any output in the closure, include the derivation that
|
||||
* produced it.
|
||||
* If flip_direction is true: for any derivation in the closure, include its outputs.
|
||||
* @param[in] callback The function to call for every store path, in no particular order
|
||||
* @param[in] userdata The userdata to pass to the callback
|
||||
*/
|
||||
nix_err nix_store_get_fs_closure(
|
||||
nix_c_context * context,
|
||||
Store * store,
|
||||
const StorePath * store_path,
|
||||
bool flip_direction,
|
||||
bool include_outputs,
|
||||
bool include_derivers,
|
||||
void * userdata,
|
||||
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path));
|
||||
|
||||
// cffi end
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "nix/store/content-address.hh"
|
||||
#include "nix/util/tests/json-characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -8,33 +9,93 @@ namespace nix {
|
||||
* ContentAddressMethod::parse, ContentAddressMethod::render
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
TEST(ContentAddressMethod, testRoundTripPrintParse_1)
|
||||
static auto methods = ::testing::Values(
|
||||
std::pair{ContentAddressMethod::Raw::Text, "text"},
|
||||
std::pair{ContentAddressMethod::Raw::Flat, "flat"},
|
||||
std::pair{ContentAddressMethod::Raw::NixArchive, "nar"},
|
||||
std::pair{ContentAddressMethod::Raw::Git, "git"});
|
||||
|
||||
struct ContentAddressMethodTest : ::testing::Test,
|
||||
::testing::WithParamInterface<std::pair<ContentAddressMethod, std::string_view>>
|
||||
{};
|
||||
|
||||
TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1)
|
||||
{
|
||||
for (ContentAddressMethod cam : {
|
||||
ContentAddressMethod::Raw::Text,
|
||||
ContentAddressMethod::Raw::Flat,
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
ContentAddressMethod::Raw::Git,
|
||||
}) {
|
||||
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
|
||||
}
|
||||
auto & [cam, _] = GetParam();
|
||||
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
|
||||
}
|
||||
|
||||
TEST(ContentAddressMethod, testRoundTripPrintParse_2)
|
||||
TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2)
|
||||
{
|
||||
for (const std::string_view camS : {
|
||||
"text",
|
||||
"flat",
|
||||
"nar",
|
||||
"git",
|
||||
}) {
|
||||
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
|
||||
}
|
||||
auto & [cam, camS] = GetParam();
|
||||
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods);
|
||||
|
||||
TEST(ContentAddressMethod, testParseContentAddressMethodOptException)
|
||||
{
|
||||
EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError);
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
* JSON
|
||||
* --------------------------------------------------------------------------*/
|
||||
|
||||
class ContentAddressTest : public virtual CharacterizationTest
|
||||
{
|
||||
std::filesystem::path unitTestData = getUnitTestData() / "content-address";
|
||||
|
||||
public:
|
||||
|
||||
/**
|
||||
* We set these in tests rather than the regular globals so we don't have
|
||||
* to worry about race conditions if the tests run concurrently.
|
||||
*/
|
||||
ExperimentalFeatureSettings mockXpSettings;
|
||||
|
||||
std::filesystem::path goldenMaster(std::string_view testStem) const override
|
||||
{
|
||||
return unitTestData / testStem;
|
||||
}
|
||||
};
|
||||
|
||||
using nlohmann::json;
|
||||
|
||||
struct ContentAddressJsonTest : ContentAddressTest,
|
||||
JsonCharacterizationTest<ContentAddress>,
|
||||
::testing::WithParamInterface<std::pair<std::string_view, ContentAddress>>
|
||||
{};
|
||||
|
||||
TEST_P(ContentAddressJsonTest, from_json)
|
||||
{
|
||||
auto & [name, expected] = GetParam();
|
||||
readJsonTest(name, expected);
|
||||
}
|
||||
|
||||
TEST_P(ContentAddressJsonTest, to_json)
|
||||
{
|
||||
auto & [name, value] = GetParam();
|
||||
writeJsonTest(name, value);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
ContentAddressJSON,
|
||||
ContentAddressJsonTest,
|
||||
::testing::Values(
|
||||
std::pair{
|
||||
"text",
|
||||
ContentAddress{
|
||||
.method = ContentAddressMethod::Raw::Text,
|
||||
.hash = hashString(HashAlgorithm::SHA256, "asdf"),
|
||||
},
|
||||
},
|
||||
std::pair{
|
||||
"nar",
|
||||
ContentAddress{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hash = hashString(HashAlgorithm::SHA256, "qwer"),
|
||||
},
|
||||
}));
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"hash": {
|
||||
"algorithm": "sha256",
|
||||
"format": "base64",
|
||||
"hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM="
|
||||
},
|
||||
"method": "nar"
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"hash": {
|
||||
"algorithm": "sha256",
|
||||
"format": "base64",
|
||||
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
|
||||
},
|
||||
"method": "text"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
local://?root=/foo bar/baz
|
||||
local://?root=/foo%20bar/baz
|
||||
@@ -0,0 +1 @@
|
||||
local://?root=/foo bar/baz
|
||||
@@ -218,6 +218,70 @@ struct LambdaAdapter
|
||||
}
|
||||
};
|
||||
|
||||
class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base
|
||||
{
|
||||
public:
|
||||
StorePath * drvPath = nullptr;
|
||||
nix_derivation * drv = nullptr;
|
||||
Store * store = nullptr;
|
||||
StorePath * outPath = nullptr;
|
||||
|
||||
void SetUp() override
|
||||
{
|
||||
nix_api_store_test_base::SetUp();
|
||||
|
||||
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
nix::settings.substituters = {};
|
||||
|
||||
store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
|
||||
// Replace the hardcoded system with the current system
|
||||
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
|
||||
|
||||
drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
drvPath = nix_add_derivation(ctx, store, drv);
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drvPath, nullptr);
|
||||
|
||||
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath_) {
|
||||
ASSERT_NE(outname, nullptr) << "Output name should not be NULL";
|
||||
auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath_);
|
||||
ASSERT_EQ(is_valid_path, true);
|
||||
ASSERT_STREQ(outname, "out") << "Expected single 'out' output";
|
||||
ASSERT_EQ(outPath, nullptr) << "Output path callback should only be called once";
|
||||
outPath = nix_store_path_clone(outPath_);
|
||||
}};
|
||||
|
||||
auto ret = nix_store_realise(
|
||||
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
ASSERT_NE(outPath, nullptr) << "Derivation should have produced an output";
|
||||
}
|
||||
|
||||
void TearDown() override
|
||||
{
|
||||
if (drvPath)
|
||||
nix_store_path_free(drvPath);
|
||||
if (outPath)
|
||||
nix_store_path_free(outPath);
|
||||
if (drv)
|
||||
nix_derivation_free(drv);
|
||||
if (store)
|
||||
nix_store_free(store);
|
||||
|
||||
nix_api_store_test_base::TearDown();
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(nix_api_store_test_base, build_from_json)
|
||||
{
|
||||
// FIXME get rid of these
|
||||
@@ -232,7 +296,10 @@ TEST_F(nix_api_store_test_base, build_from_json)
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
|
||||
auto * drv = nix_derivation_from_json(ctx, store, buffer.str().c_str());
|
||||
// Replace the hardcoded system with the current system
|
||||
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
|
||||
|
||||
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
@@ -240,15 +307,21 @@ TEST_F(nix_api_store_test_base, build_from_json)
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
int callbackCount = 0;
|
||||
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) {
|
||||
ASSERT_NE(outname, nullptr);
|
||||
ASSERT_STREQ(outname, "out");
|
||||
ASSERT_NE(outPath, nullptr);
|
||||
auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath);
|
||||
ASSERT_EQ(is_valid_path, true);
|
||||
callbackCount++;
|
||||
}};
|
||||
|
||||
auto ret = nix_store_realise(
|
||||
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
ASSERT_EQ(callbackCount, 1) << "Callback should have been invoked exactly once";
|
||||
|
||||
// Clean up
|
||||
nix_store_path_free(drvPath);
|
||||
@@ -256,4 +329,468 @@ TEST_F(nix_api_store_test_base, build_from_json)
|
||||
nix_store_free(store);
|
||||
}
|
||||
|
||||
TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
|
||||
{
|
||||
// Test that nix_store_realise properly reports errors when the system is invalid
|
||||
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
nix::settings.substituters = {};
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
|
||||
// Use an invalid system that cannot be built
|
||||
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", "bogus65-bogusos");
|
||||
|
||||
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
auto * drvPath = nix_add_derivation(ctx, store, drv);
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drvPath, nullptr);
|
||||
|
||||
int callbackCount = 0;
|
||||
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }};
|
||||
|
||||
auto ret = nix_store_realise(
|
||||
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
|
||||
|
||||
// Should fail with an error
|
||||
ASSERT_NE(ret, NIX_OK);
|
||||
ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build fails";
|
||||
|
||||
// Check that error message is set
|
||||
std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
|
||||
ASSERT_FALSE(errMsg.empty()) << "Error message should be set";
|
||||
ASSERT_NE(errMsg.find("system"), std::string::npos) << "Error should mention system";
|
||||
|
||||
// Clean up
|
||||
nix_store_path_free(drvPath);
|
||||
nix_derivation_free(drv);
|
||||
nix_store_free(store);
|
||||
}
|
||||
|
||||
TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
|
||||
{
|
||||
// Test that nix_store_realise properly reports errors when the builder fails
|
||||
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
nix::settings.substituters = {};
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
|
||||
// Replace with current system and make builder command fail
|
||||
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
|
||||
jsonStr = nix::replaceStrings(jsonStr, "echo $name foo > $out", "exit 1");
|
||||
|
||||
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
auto * drvPath = nix_add_derivation(ctx, store, drv);
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drvPath, nullptr);
|
||||
|
||||
int callbackCount = 0;
|
||||
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }};
|
||||
|
||||
auto ret = nix_store_realise(
|
||||
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
|
||||
|
||||
// Should fail with an error
|
||||
ASSERT_NE(ret, NIX_OK);
|
||||
ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build fails";
|
||||
|
||||
// Check that error message is set
|
||||
std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
|
||||
ASSERT_FALSE(errMsg.empty()) << "Error message should be set";
|
||||
|
||||
// Clean up
|
||||
nix_store_path_free(drvPath);
|
||||
nix_derivation_free(drv);
|
||||
nix_store_free(store);
|
||||
}
|
||||
|
||||
TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
|
||||
{
|
||||
// Test that nix_store_realise properly reports errors when builder succeeds but produces no output
|
||||
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
nix::settings.substituters = {};
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
|
||||
// Replace with current system and make builder succeed but not produce output
|
||||
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
|
||||
jsonStr = nix::replaceStrings(jsonStr, "echo $name foo > $out", "true");
|
||||
|
||||
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
auto * drvPath = nix_add_derivation(ctx, store, drv);
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drvPath, nullptr);
|
||||
|
||||
int callbackCount = 0;
|
||||
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }};
|
||||
|
||||
auto ret = nix_store_realise(
|
||||
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
|
||||
|
||||
// Should fail with an error
|
||||
ASSERT_NE(ret, NIX_OK);
|
||||
ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build produces no output";
|
||||
|
||||
// Check that error message is set
|
||||
std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
|
||||
ASSERT_FALSE(errMsg.empty()) << "Error message should be set";
|
||||
|
||||
// Clean up
|
||||
nix_store_path_free(drvPath);
|
||||
nix_derivation_free(drv);
|
||||
nix_store_free(store);
|
||||
}
|
||||
|
||||
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_with_outputs)
|
||||
{
|
||||
// Test closure computation with include_outputs on a derivation path
|
||||
struct CallbackData
|
||||
{
|
||||
std::set<std::string> * paths;
|
||||
};
|
||||
|
||||
std::set<std::string> closure_paths;
|
||||
CallbackData data{&closure_paths};
|
||||
|
||||
auto ret = nix_store_get_fs_closure(
|
||||
ctx,
|
||||
store,
|
||||
drvPath, // Use derivation path
|
||||
false, // flip_direction
|
||||
true, // include_outputs - include the outputs in the closure
|
||||
false, // include_derivers
|
||||
&data,
|
||||
[](nix_c_context * context, void * userdata, const StorePath * path) {
|
||||
auto * data = static_cast<CallbackData *>(userdata);
|
||||
std::string path_str;
|
||||
nix_store_path_name(path, OBSERVE_STRING(path_str));
|
||||
auto [it, inserted] = data->paths->insert(path_str);
|
||||
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
|
||||
});
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
|
||||
// The closure should contain the derivation and its outputs
|
||||
ASSERT_GE(closure_paths.size(), 2);
|
||||
|
||||
// Verify the output path is in the closure
|
||||
std::string outPathName;
|
||||
nix_store_path_name(outPath, OBSERVE_STRING(outPathName));
|
||||
ASSERT_EQ(closure_paths.count(outPathName), 1);
|
||||
}
|
||||
|
||||
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_without_outputs)
|
||||
{
|
||||
// Test closure computation WITHOUT include_outputs on a derivation path
|
||||
struct CallbackData
|
||||
{
|
||||
std::set<std::string> * paths;
|
||||
};
|
||||
|
||||
std::set<std::string> closure_paths;
|
||||
CallbackData data{&closure_paths};
|
||||
|
||||
auto ret = nix_store_get_fs_closure(
|
||||
ctx,
|
||||
store,
|
||||
drvPath, // Use derivation path
|
||||
false, // flip_direction
|
||||
false, // include_outputs - do NOT include the outputs
|
||||
false, // include_derivers
|
||||
&data,
|
||||
[](nix_c_context * context, void * userdata, const StorePath * path) {
|
||||
auto * data = static_cast<CallbackData *>(userdata);
|
||||
std::string path_str;
|
||||
nix_store_path_name(path, OBSERVE_STRING(path_str));
|
||||
auto [it, inserted] = data->paths->insert(path_str);
|
||||
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
|
||||
});
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
|
||||
// Verify the output path is NOT in the closure
|
||||
std::string outPathName;
|
||||
nix_store_path_name(outPath, OBSERVE_STRING(outPathName));
|
||||
ASSERT_EQ(closure_paths.count(outPathName), 0) << "Output path should not be in closure when includeOutputs=false";
|
||||
}
|
||||
|
||||
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_flip_direction)
|
||||
{
|
||||
// Test closure computation with flip_direction on a derivation path
|
||||
// When flip_direction=true, we get the reverse dependencies (what depends on this path)
|
||||
// For a derivation, this should NOT include outputs even with include_outputs=true
|
||||
struct CallbackData
|
||||
{
|
||||
std::set<std::string> * paths;
|
||||
};
|
||||
|
||||
std::set<std::string> closure_paths;
|
||||
CallbackData data{&closure_paths};
|
||||
|
||||
auto ret = nix_store_get_fs_closure(
|
||||
ctx,
|
||||
store,
|
||||
drvPath, // Use derivation path
|
||||
true, // flip_direction - get reverse dependencies
|
||||
true, // include_outputs
|
||||
false, // include_derivers
|
||||
&data,
|
||||
[](nix_c_context * context, void * userdata, const StorePath * path) {
|
||||
auto * data = static_cast<CallbackData *>(userdata);
|
||||
std::string path_str;
|
||||
nix_store_path_name(path, OBSERVE_STRING(path_str));
|
||||
auto [it, inserted] = data->paths->insert(path_str);
|
||||
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
|
||||
});
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
|
||||
// Verify the output path is NOT in the closure when direction is flipped
|
||||
std::string outPathName;
|
||||
nix_store_path_name(outPath, OBSERVE_STRING(outPathName));
|
||||
ASSERT_EQ(closure_paths.count(outPathName), 0) << "Output path should not be in closure when flip_direction=true";
|
||||
}
|
||||
|
||||
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_include_derivers)
|
||||
{
|
||||
// Test closure computation with include_derivers on an output path
|
||||
// This should include the derivation that produced the output
|
||||
struct CallbackData
|
||||
{
|
||||
std::set<std::string> * paths;
|
||||
};
|
||||
|
||||
std::set<std::string> closure_paths;
|
||||
CallbackData data{&closure_paths};
|
||||
|
||||
auto ret = nix_store_get_fs_closure(
|
||||
ctx,
|
||||
store,
|
||||
outPath, // Use output path (not derivation)
|
||||
false, // flip_direction
|
||||
false, // include_outputs
|
||||
true, // include_derivers - include the derivation
|
||||
&data,
|
||||
[](nix_c_context * context, void * userdata, const StorePath * path) {
|
||||
auto * data = static_cast<CallbackData *>(userdata);
|
||||
std::string path_str;
|
||||
nix_store_path_name(path, OBSERVE_STRING(path_str));
|
||||
auto [it, inserted] = data->paths->insert(path_str);
|
||||
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
|
||||
});
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
|
||||
// Verify the derivation path is in the closure
|
||||
// Deriver is nasty stateful, and this assertion is only guaranteed because
|
||||
// we're using an empty store as our starting point. Otherwise, if the
|
||||
// output happens to exist, the deriver could be anything.
|
||||
std::string drvPathName;
|
||||
nix_store_path_name(drvPath, OBSERVE_STRING(drvPathName));
|
||||
ASSERT_EQ(closure_paths.count(drvPathName), 1) << "Derivation should be in closure when include_derivers=true";
|
||||
}
|
||||
|
||||
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
|
||||
{
|
||||
// Test that nix_store_realise returns outputs in alphabetical order by output name.
|
||||
// This test uses a CA derivation with 10 outputs in randomized input order
|
||||
// to verify that the callback order is deterministic and alphabetical.
|
||||
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
nix::settings.substituters = {};
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
// Create a CA derivation with 10 outputs using proper placeholders
|
||||
auto outa_ph = nix::hashPlaceholder("outa");
|
||||
auto outb_ph = nix::hashPlaceholder("outb");
|
||||
auto outc_ph = nix::hashPlaceholder("outc");
|
||||
auto outd_ph = nix::hashPlaceholder("outd");
|
||||
auto oute_ph = nix::hashPlaceholder("oute");
|
||||
auto outf_ph = nix::hashPlaceholder("outf");
|
||||
auto outg_ph = nix::hashPlaceholder("outg");
|
||||
auto outh_ph = nix::hashPlaceholder("outh");
|
||||
auto outi_ph = nix::hashPlaceholder("outi");
|
||||
auto outj_ph = nix::hashPlaceholder("outj");
|
||||
|
||||
std::string drvJson = R"({
|
||||
"version": 3,
|
||||
"name": "multi-output-test",
|
||||
"system": ")" + nix::settings.thisSystem.get()
|
||||
+ R"(",
|
||||
"builder": "/bin/sh",
|
||||
"args": ["-c", "echo a > $outa; echo b > $outb; echo c > $outc; echo d > $outd; echo e > $oute; echo f > $outf; echo g > $outg; echo h > $outh; echo i > $outi; echo j > $outj"],
|
||||
"env": {
|
||||
"builder": "/bin/sh",
|
||||
"name": "multi-output-test",
|
||||
"system": ")" + nix::settings.thisSystem.get()
|
||||
+ R"(",
|
||||
"outf": ")" + outf_ph
|
||||
+ R"(",
|
||||
"outd": ")" + outd_ph
|
||||
+ R"(",
|
||||
"outi": ")" + outi_ph
|
||||
+ R"(",
|
||||
"oute": ")" + oute_ph
|
||||
+ R"(",
|
||||
"outh": ")" + outh_ph
|
||||
+ R"(",
|
||||
"outc": ")" + outc_ph
|
||||
+ R"(",
|
||||
"outb": ")" + outb_ph
|
||||
+ R"(",
|
||||
"outg": ")" + outg_ph
|
||||
+ R"(",
|
||||
"outj": ")" + outj_ph
|
||||
+ R"(",
|
||||
"outa": ")" + outa_ph
|
||||
+ R"("
|
||||
},
|
||||
"inputDrvs": {},
|
||||
"inputSrcs": [],
|
||||
"outputs": {
|
||||
"outd": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outf": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outg": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outb": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outc": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outi": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outj": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outh": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"outa": { "hashAlgo": "sha256", "method": "nar" },
|
||||
"oute": { "hashAlgo": "sha256", "method": "nar" }
|
||||
}
|
||||
})";
|
||||
|
||||
auto * drv = nix_derivation_from_json(ctx, store, drvJson.c_str());
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drv, nullptr);
|
||||
|
||||
auto * drvPath = nix_add_derivation(ctx, store, drv);
|
||||
assert_ctx_ok();
|
||||
ASSERT_NE(drvPath, nullptr);
|
||||
|
||||
// Realise the derivation - capture the order outputs are returned
|
||||
std::map<std::string, nix::StorePath> outputs;
|
||||
std::vector<std::string> output_order;
|
||||
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) {
|
||||
ASSERT_NE(outname, nullptr);
|
||||
ASSERT_NE(outPath, nullptr);
|
||||
output_order.push_back(outname);
|
||||
outputs.emplace(outname, outPath->path);
|
||||
}};
|
||||
|
||||
auto ret = nix_store_realise(
|
||||
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
ASSERT_EQ(outputs.size(), 10);
|
||||
|
||||
// Verify outputs are returned in alphabetical order by output name
|
||||
std::vector<std::string> expected_order = {
|
||||
"outa", "outb", "outc", "outd", "oute", "outf", "outg", "outh", "outi", "outj"};
|
||||
ASSERT_EQ(output_order, expected_order) << "Outputs should be returned in alphabetical order by output name";
|
||||
|
||||
// Now compute closure with include_outputs and collect paths in order
|
||||
struct CallbackData
|
||||
{
|
||||
std::vector<std::string> * paths;
|
||||
};
|
||||
|
||||
std::vector<std::string> closure_paths;
|
||||
CallbackData data{&closure_paths};
|
||||
|
||||
ret = nix_store_get_fs_closure(
|
||||
ctx,
|
||||
store,
|
||||
drvPath,
|
||||
false, // flip_direction
|
||||
true, // include_outputs - include the outputs in the closure
|
||||
false, // include_derivers
|
||||
&data,
|
||||
[](nix_c_context * context, void * userdata, const StorePath * path) {
|
||||
auto * data = static_cast<CallbackData *>(userdata);
|
||||
std::string path_str;
|
||||
nix_store_path_name(path, OBSERVE_STRING(path_str));
|
||||
data->paths->push_back(path_str);
|
||||
});
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ(ret, NIX_OK);
|
||||
|
||||
// Should contain at least the derivation and 10 outputs
|
||||
ASSERT_GE(closure_paths.size(), 11);
|
||||
|
||||
// Verify all outputs are present in the closure
|
||||
for (const auto & [outname, outPath] : outputs) {
|
||||
std::string outPathName = store->ptr->printStorePath(outPath);
|
||||
|
||||
bool found = false;
|
||||
for (const auto & p : closure_paths) {
|
||||
// nix_store_path_name returns just the name part, so match against full path name
|
||||
if (outPathName.find(p) != std::string::npos) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ASSERT_TRUE(found) << "Output " << outname << " (" << outPathName << ") not found in closure";
|
||||
}
|
||||
|
||||
nix_store_path_free(drvPath);
|
||||
nix_derivation_free(drv);
|
||||
nix_store_free(store);
|
||||
}
|
||||
|
||||
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagation)
|
||||
{
|
||||
// Test that errors in the callback abort the closure computation
|
||||
struct CallbackData
|
||||
{
|
||||
int * count;
|
||||
};
|
||||
|
||||
int call_count = 0;
|
||||
CallbackData data{&call_count};
|
||||
|
||||
auto ret = nix_store_get_fs_closure(
|
||||
ctx,
|
||||
store,
|
||||
drvPath, // Use derivation path
|
||||
false, // flip_direction
|
||||
true, // include_outputs
|
||||
false, // include_derivers
|
||||
&data,
|
||||
[](nix_c_context * context, void * userdata, const StorePath * path) {
|
||||
auto * data = static_cast<CallbackData *>(userdata);
|
||||
(*data->count)++;
|
||||
// Set an error immediately
|
||||
nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Test error");
|
||||
});
|
||||
|
||||
// Should have aborted with error
|
||||
ASSERT_EQ(ret, NIX_ERR_UNKNOWN);
|
||||
ASSERT_EQ(call_count, 1); // Should have been called exactly once, then aborted
|
||||
}
|
||||
|
||||
} // namespace nixC
|
||||
@@ -1,4 +1,6 @@
|
||||
#include "nix/store/references.hh"
|
||||
#include "nix/store/path-references.hh"
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
@@ -79,4 +81,145 @@ TEST(references, scan)
|
||||
}
|
||||
}
|
||||
|
||||
TEST(references, scanForReferencesDeep)
|
||||
{
|
||||
using File = MemorySourceAccessor::File;
|
||||
|
||||
// Create store paths to search for
|
||||
StorePath path1{"dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo"};
|
||||
StorePath path2{"zc842j0rz61mjsp3h3wp5ly71ak6qgdn-bar"};
|
||||
StorePath path3{"a5cn2i4b83gnsm60d38l3kgb8qfplm11-baz"};
|
||||
|
||||
StorePathSet refs{path1, path2, path3};
|
||||
|
||||
std::string_view hash1 = path1.hashPart();
|
||||
std::string_view hash2 = path2.hashPart();
|
||||
std::string_view hash3 = path3.hashPart();
|
||||
|
||||
// Create an in-memory file system with various reference patterns
|
||||
auto accessor = make_ref<MemorySourceAccessor>();
|
||||
accessor->root = File::Directory{
|
||||
.contents{
|
||||
{
|
||||
// file1.txt: contains hash1
|
||||
"file1.txt",
|
||||
File::Regular{
|
||||
.contents = "This file references " + hash1 + " in its content",
|
||||
},
|
||||
},
|
||||
{
|
||||
// file2.txt: contains hash2 and hash3
|
||||
"file2.txt",
|
||||
File::Regular{
|
||||
.contents = "Multiple refs: " + hash2 + " and also " + hash3,
|
||||
},
|
||||
},
|
||||
{
|
||||
// file3.txt: contains no references
|
||||
"file3.txt",
|
||||
File::Regular{
|
||||
.contents = "This file has no store path references at all",
|
||||
},
|
||||
},
|
||||
{
|
||||
// subdir: a subdirectory
|
||||
"subdir",
|
||||
File::Directory{
|
||||
.contents{
|
||||
{
|
||||
// subdir/file4.txt: contains hash1 again
|
||||
"file4.txt",
|
||||
File::Regular{
|
||||
.contents = "Subdirectory file with " + hash1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
// link1: a symlink that contains a reference in its target
|
||||
"link1",
|
||||
File::Symlink{
|
||||
.target = hash2 + "-target",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
// Test the callback-based API
|
||||
{
|
||||
std::map<CanonPath, StorePathSet> foundRefs;
|
||||
|
||||
scanForReferencesDeep(*accessor, CanonPath::root, refs, [&](FileRefScanResult result) {
|
||||
foundRefs[std::move(result.filePath)] = std::move(result.foundRefs);
|
||||
});
|
||||
|
||||
// Verify we found the expected references
|
||||
EXPECT_EQ(foundRefs.size(), 4); // file1, file2, file4, link1
|
||||
|
||||
// Check file1.txt found path1
|
||||
{
|
||||
CanonPath f1Path("/file1.txt");
|
||||
auto it = foundRefs.find(f1Path);
|
||||
ASSERT_TRUE(it != foundRefs.end());
|
||||
EXPECT_EQ(it->second.size(), 1);
|
||||
EXPECT_TRUE(it->second.count(path1));
|
||||
}
|
||||
|
||||
// Check file2.txt found path2 and path3
|
||||
{
|
||||
CanonPath f2Path("/file2.txt");
|
||||
auto it = foundRefs.find(f2Path);
|
||||
ASSERT_TRUE(it != foundRefs.end());
|
||||
EXPECT_EQ(it->second.size(), 2);
|
||||
EXPECT_TRUE(it->second.count(path2));
|
||||
EXPECT_TRUE(it->second.count(path3));
|
||||
}
|
||||
|
||||
// Check file3.txt is not in results (no refs)
|
||||
{
|
||||
CanonPath f3Path("/file3.txt");
|
||||
EXPECT_FALSE(foundRefs.count(f3Path));
|
||||
}
|
||||
|
||||
// Check subdir/file4.txt found path1
|
||||
{
|
||||
CanonPath f4Path("/subdir/file4.txt");
|
||||
auto it = foundRefs.find(f4Path);
|
||||
ASSERT_TRUE(it != foundRefs.end());
|
||||
EXPECT_EQ(it->second.size(), 1);
|
||||
EXPECT_TRUE(it->second.count(path1));
|
||||
}
|
||||
|
||||
// Check symlink found path2
|
||||
{
|
||||
CanonPath linkPath("/link1");
|
||||
auto it = foundRefs.find(linkPath);
|
||||
ASSERT_TRUE(it != foundRefs.end());
|
||||
EXPECT_EQ(it->second.size(), 1);
|
||||
EXPECT_TRUE(it->second.count(path2));
|
||||
}
|
||||
}
|
||||
|
||||
// Test the map-based convenience API
|
||||
{
|
||||
auto results = scanForReferencesDeep(*accessor, CanonPath::root, refs);
|
||||
|
||||
EXPECT_EQ(results.size(), 4); // file1, file2, file4, link1
|
||||
|
||||
// Verify all expected files are in the results
|
||||
EXPECT_TRUE(results.count(CanonPath("/file1.txt")));
|
||||
EXPECT_TRUE(results.count(CanonPath("/file2.txt")));
|
||||
EXPECT_TRUE(results.count(CanonPath("/subdir/file4.txt")));
|
||||
EXPECT_TRUE(results.count(CanonPath("/link1")));
|
||||
EXPECT_FALSE(results.count(CanonPath("/file3.txt")));
|
||||
|
||||
// Verify the references found in each file are correct
|
||||
EXPECT_EQ(results.at(CanonPath("/file1.txt")), StorePathSet{path1});
|
||||
EXPECT_EQ(results.at(CanonPath("/file2.txt")), StorePathSet({path2, path3}));
|
||||
EXPECT_EQ(results.at(CanonPath("/subdir/file4.txt")), StorePathSet{path1});
|
||||
EXPECT_EQ(results.at(CanonPath("/link1")), StorePathSet{path2});
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -70,6 +70,25 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
"with_profile_and_region",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://my-bucket/my-key.txt?versionId=abc123xyz",
|
||||
{
|
||||
.bucket = "my-bucket",
|
||||
.key = {"my-key.txt"},
|
||||
.versionId = "abc123xyz",
|
||||
},
|
||||
"with_versionId",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/path/to/object?region=eu-west-1&versionId=version456",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"path", "to", "object"},
|
||||
.region = "eu-west-1",
|
||||
.versionId = "version456",
|
||||
},
|
||||
"with_region_and_versionId",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?endpoint=https://minio.local&scheme=http",
|
||||
{
|
||||
@@ -222,6 +241,37 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
|
||||
"complex_path_and_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
.key = {"my-key.txt"},
|
||||
.versionId = "abc123xyz",
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
.query = {{"versionId", "abc123xyz"}},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
|
||||
"with_versionId",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "versioned-bucket",
|
||||
.key = {"path", "to", "object"},
|
||||
.region = "eu-west-1",
|
||||
.versionId = "version456",
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "versioned-bucket", "path", "to", "object"},
|
||||
.query = {{"versionId", "version456"}},
|
||||
},
|
||||
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
|
||||
"with_region_and_versionId",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
||||
|
||||
|
||||
@@ -100,9 +100,12 @@ URI_TEST(local_1, localExample_1)
|
||||
|
||||
URI_TEST(local_2, localExample_2)
|
||||
|
||||
/* Test path with spaces */
|
||||
/* Test path with encoded spaces */
|
||||
URI_TEST(local_3, localExample_3)
|
||||
|
||||
/* Test path with spaces that are improperly not encoded */
|
||||
URI_TEST_READ(local_3_no_percent, localExample_3)
|
||||
|
||||
URI_TEST_READ(local_shorthand_1, localExample_1)
|
||||
|
||||
URI_TEST_READ(local_shorthand_2, localExample_2)
|
||||
|
||||
@@ -76,9 +76,10 @@ std::optional<std::string> BinaryCacheStore::getNixCacheInfo()
|
||||
return getFile(cacheInfoFile);
|
||||
}
|
||||
|
||||
void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType)
|
||||
void BinaryCacheStore::upsertFile(
|
||||
const std::string & path, std::string && data, const std::string & mimeType, uint64_t sizeHint)
|
||||
{
|
||||
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
|
||||
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType, sizeHint);
|
||||
}
|
||||
|
||||
void BinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
|
||||
@@ -274,7 +275,8 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
upsertFile(
|
||||
narInfo->url,
|
||||
std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
|
||||
"application/x-nix-nar");
|
||||
"application/x-nix-nar",
|
||||
narInfo->fileSize);
|
||||
} else
|
||||
stats.narWriteAverted++;
|
||||
|
||||
|
||||
@@ -239,7 +239,6 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
fmt("building '%s'", worker.store.printStorePath(drvPath));
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook)
|
||||
msg += fmt(" on '%s'", hook->machineName);
|
||||
@@ -678,7 +677,15 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
{
|
||||
builder.reset();
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, output] : builtOutputs) {
|
||||
/* In the check case we install no store objects, and so
|
||||
`builtOutputs` is empty. However, per issue #14287, there is
|
||||
an expectation that the post-build hook is still executed.
|
||||
(This is useful for e.g. logging successful deterministic rebuilds.)
|
||||
|
||||
In order to make that work, in the check case just load the
|
||||
(preexisting) infos from scratch, rather than relying on what
|
||||
`DerivationBuilder` returned to us. */
|
||||
for (auto & [_, output] : buildMode == bmCheck ? checkPathValidity(initialOutputs).second : builtOutputs) {
|
||||
// for sake of `bmRepair`
|
||||
worker.markContentsGood(output.outPath);
|
||||
outputPaths.insert(output.outPath);
|
||||
|
||||
@@ -25,7 +25,7 @@ DesugaredEnv DesugaredEnv::create(
|
||||
if (drv.structuredAttrs) {
|
||||
auto json = drv.structuredAttrs->prepareStructuredAttrs(store, drvOptions, inputPaths, drv.outputs);
|
||||
res.atFileEnvPair("NIX_ATTRS_SH_FILE", ".attrs.sh") = StructuredAttrs::writeShell(json);
|
||||
res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = json.dump();
|
||||
res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = static_cast<nlohmann::json>(std::move(json)).dump();
|
||||
} else {
|
||||
/* In non-structured mode, set all bindings either directory in the
|
||||
environment or via a file, as specified by
|
||||
|
||||
@@ -268,16 +268,18 @@ Goal::Co PathSubstitutionGoal::tryToRun(
|
||||
try {
|
||||
promise.get_future().get();
|
||||
} catch (std::exception & e) {
|
||||
printError(e.what());
|
||||
|
||||
/* Cause the parent build to fail unless --fallback is given,
|
||||
or the substitute has disappeared. The latter case behaves
|
||||
the same as the substitute never having existed in the
|
||||
first place. */
|
||||
try {
|
||||
throw;
|
||||
} catch (SubstituteGone &) {
|
||||
} catch (SubstituteGone & sg) {
|
||||
/* Missing NARs are expected when they've been garbage collected.
|
||||
This is not a failure, so log as a warning instead of an error. */
|
||||
logWarning({.msg = sg.info().msg});
|
||||
} catch (...) {
|
||||
printError(e.what());
|
||||
substituterFailed = true;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "nix/util/args.hh"
|
||||
#include "nix/store/content-address.hh"
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -300,3 +301,36 @@ Hash ContentAddressWithReferences::getHash() const
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
using namespace nix;
|
||||
|
||||
ContentAddressMethod adl_serializer<ContentAddressMethod>::from_json(const json & json)
|
||||
{
|
||||
return ContentAddressMethod::parse(getString(json));
|
||||
}
|
||||
|
||||
void adl_serializer<ContentAddressMethod>::to_json(json & json, const ContentAddressMethod & m)
|
||||
{
|
||||
json = m.render();
|
||||
}
|
||||
|
||||
ContentAddress adl_serializer<ContentAddress>::from_json(const json & json)
|
||||
{
|
||||
auto obj = getObject(json);
|
||||
return {
|
||||
.method = adl_serializer<ContentAddressMethod>::from_json(valueAt(obj, "method")),
|
||||
.hash = valueAt(obj, "hash"),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<ContentAddress>::to_json(json & json, const ContentAddress & ca)
|
||||
{
|
||||
json = {
|
||||
{"method", ca.method},
|
||||
{"hash", ca.hash},
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
@@ -1031,7 +1031,7 @@ void processConnection(ref<Store> store, FdSource && from, FdSink && to, Trusted
|
||||
auto [protoVersion, features] =
|
||||
WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, WorkerProto::allFeatures);
|
||||
|
||||
if (protoVersion < 256 + 18)
|
||||
if (protoVersion < MINIMUM_PROTOCOL_VERSION)
|
||||
throw Error("the Nix client version is too old");
|
||||
|
||||
WorkerProto::BasicServerConnection conn;
|
||||
|
||||
@@ -22,9 +22,9 @@ getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::
|
||||
if (i == parsed->structuredAttrs.end())
|
||||
return {};
|
||||
else {
|
||||
if (!i->is_string())
|
||||
if (!i->second.is_string())
|
||||
throw Error("attribute '%s' of must be a string", name);
|
||||
return i->get<std::string>();
|
||||
return i->second.get<std::string>();
|
||||
}
|
||||
} else {
|
||||
auto i = env.find(name);
|
||||
@@ -42,9 +42,9 @@ static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, c
|
||||
if (i == parsed->structuredAttrs.end())
|
||||
return def;
|
||||
else {
|
||||
if (!i->is_boolean())
|
||||
if (!i->second.is_boolean())
|
||||
throw Error("attribute '%s' must be a Boolean", name);
|
||||
return i->get<bool>();
|
||||
return i->second.get<bool>();
|
||||
}
|
||||
} else {
|
||||
auto i = env.find(name);
|
||||
@@ -63,10 +63,11 @@ getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std:
|
||||
if (i == parsed->structuredAttrs.end())
|
||||
return {};
|
||||
else {
|
||||
if (!i->is_array())
|
||||
if (!i->second.is_array())
|
||||
throw Error("attribute '%s' must be a list of strings", name);
|
||||
auto & a = getArray(i->second);
|
||||
Strings res;
|
||||
for (auto j = i->begin(); j != i->end(); ++j) {
|
||||
for (auto j = a.begin(); j != a.end(); ++j) {
|
||||
if (!j->is_string())
|
||||
throw Error("attribute '%s' must be a list of strings", name);
|
||||
res.push_back(j->get<std::string>());
|
||||
@@ -116,27 +117,29 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
|
||||
DerivationOptions defaults = {};
|
||||
|
||||
if (shouldWarn && parsed) {
|
||||
if (get(parsed->structuredAttrs, "allowedReferences")) {
|
||||
auto & structuredAttrs = parsed->structuredAttrs;
|
||||
|
||||
if (get(structuredAttrs, "allowedReferences")) {
|
||||
warn(
|
||||
"'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
|
||||
}
|
||||
if (get(parsed->structuredAttrs, "allowedRequisites")) {
|
||||
if (get(structuredAttrs, "allowedRequisites")) {
|
||||
warn(
|
||||
"'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
|
||||
}
|
||||
if (get(parsed->structuredAttrs, "disallowedRequisites")) {
|
||||
if (get(structuredAttrs, "disallowedRequisites")) {
|
||||
warn(
|
||||
"'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
|
||||
}
|
||||
if (get(parsed->structuredAttrs, "disallowedReferences")) {
|
||||
if (get(structuredAttrs, "disallowedReferences")) {
|
||||
warn(
|
||||
"'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
|
||||
}
|
||||
if (get(parsed->structuredAttrs, "maxSize")) {
|
||||
if (get(structuredAttrs, "maxSize")) {
|
||||
warn(
|
||||
"'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
|
||||
}
|
||||
if (get(parsed->structuredAttrs, "maxClosureSize")) {
|
||||
if (get(structuredAttrs, "maxClosureSize")) {
|
||||
warn(
|
||||
"'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
|
||||
}
|
||||
@@ -145,11 +148,15 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
|
||||
return {
|
||||
.outputChecks = [&]() -> OutputChecksVariant {
|
||||
if (parsed) {
|
||||
auto & structuredAttrs = parsed->structuredAttrs;
|
||||
|
||||
std::map<std::string, OutputChecks> res;
|
||||
if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) {
|
||||
for (auto & [outputName, output] : getObject(*outputChecks)) {
|
||||
if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
|
||||
for (auto & [outputName, output_] : getObject(*outputChecks)) {
|
||||
OutputChecks checks;
|
||||
|
||||
auto & output = getObject(output_);
|
||||
|
||||
if (auto maxSize = get(output, "maxSize"))
|
||||
checks.maxSize = maxSize->get<uint64_t>();
|
||||
|
||||
@@ -195,7 +202,9 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
|
||||
std::map<std::string, bool> res;
|
||||
|
||||
if (parsed) {
|
||||
if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) {
|
||||
auto & structuredAttrs = parsed->structuredAttrs;
|
||||
|
||||
if (auto * udr = get(structuredAttrs, "unsafeDiscardReferences")) {
|
||||
for (auto & [outputName, output] : getObject(*udr)) {
|
||||
if (!output.is_boolean())
|
||||
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName);
|
||||
@@ -226,7 +235,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
|
||||
std::map<std::string, StringSet> ret;
|
||||
|
||||
if (parsed) {
|
||||
auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
|
||||
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
|
||||
if (!e || !e->is_object())
|
||||
return ret;
|
||||
for (auto & [key, value] : getObject(*e)) {
|
||||
@@ -333,8 +342,10 @@ namespace nlohmann {
|
||||
|
||||
using namespace nix;
|
||||
|
||||
DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json)
|
||||
DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json_)
|
||||
{
|
||||
auto & json = getObject(json_);
|
||||
|
||||
return {
|
||||
.outputChecks = [&]() -> OutputChecksVariant {
|
||||
auto outputChecks = getObject(valueAt(json, "outputChecks"));
|
||||
@@ -397,13 +408,24 @@ void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOpt
|
||||
json["allowSubstitutes"] = o.allowSubstitutes;
|
||||
}
|
||||
|
||||
DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json)
|
||||
template<typename T>
|
||||
static inline std::optional<T> ptrToOwned(const json * ptr)
|
||||
{
|
||||
if (ptr)
|
||||
return std::optional{*ptr};
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json_)
|
||||
{
|
||||
auto & json = getObject(json_);
|
||||
|
||||
return {
|
||||
.ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
|
||||
.allowedReferences = nullableValueAt(json, "allowedReferences"),
|
||||
.allowedReferences = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedReferences"))),
|
||||
.disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
|
||||
.allowedRequisites = nullableValueAt(json, "allowedRequisites"),
|
||||
.allowedRequisites = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedRequisites"))),
|
||||
.disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ struct DummyStoreImpl : DummyStore
|
||||
/**
|
||||
* The dummy store is incapable of *not* trusting! :)
|
||||
*/
|
||||
virtual std::optional<TrustedFlag> isTrustedClient() override
|
||||
std::optional<TrustedFlag> isTrustedClient() override
|
||||
{
|
||||
return Trusted;
|
||||
}
|
||||
@@ -258,18 +258,6 @@ struct DummyStoreImpl : DummyStore
|
||||
});
|
||||
}
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override
|
||||
{
|
||||
bool visited = contents.cvisit(path, [&](const auto & kv) {
|
||||
const auto & [info, accessor] = kv.second;
|
||||
SourcePath sourcePath(accessor);
|
||||
dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive);
|
||||
});
|
||||
|
||||
if (!visited)
|
||||
throw Error("path '%s' is not valid", printStorePath(path));
|
||||
}
|
||||
|
||||
void queryRealisationUncached(
|
||||
const DrvOutput & drvOutput, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
|
||||
{
|
||||
|
||||
@@ -384,28 +384,30 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (settings.downloadSpeed.get() > 0)
|
||||
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
|
||||
|
||||
if (request.head)
|
||||
if (request.method == HttpMethod::HEAD)
|
||||
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
|
||||
|
||||
if (request.method == HttpMethod::DELETE)
|
||||
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
|
||||
if (request.data) {
|
||||
if (request.post)
|
||||
if (request.method == HttpMethod::POST) {
|
||||
curl_easy_setopt(req, CURLOPT_POST, 1L);
|
||||
else
|
||||
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->length());
|
||||
} else if (request.method == HttpMethod::PUT) {
|
||||
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
|
||||
} else {
|
||||
unreachable();
|
||||
}
|
||||
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_READDATA, this);
|
||||
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
|
||||
curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_SEEKDATA, this);
|
||||
}
|
||||
|
||||
if (request.verifyTLS) {
|
||||
if (settings.caFile != "")
|
||||
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
|
||||
} else {
|
||||
curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0);
|
||||
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
}
|
||||
if (settings.caFile != "")
|
||||
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
|
||||
|
||||
#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
|
||||
curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
|
||||
@@ -622,7 +624,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
void quit()
|
||||
{
|
||||
quitting = true;
|
||||
/* We wil not be processing any more incomming requests */
|
||||
/* We wil not be processing any more incoming requests */
|
||||
while (!incoming.empty())
|
||||
incoming.pop();
|
||||
}
|
||||
@@ -922,6 +924,11 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
|
||||
return enqueueFileTransfer(request).get();
|
||||
}
|
||||
|
||||
FileTransferResult FileTransfer::deleteResource(const FileTransferRequest & request)
|
||||
{
|
||||
return enqueueFileTransfer(request).get();
|
||||
}
|
||||
|
||||
void FileTransfer::download(
|
||||
FileTransferRequest && request, Sink & sink, std::function<void(FileTransferResult)> resultCallback)
|
||||
{
|
||||
|
||||
+2
-3
@@ -5,6 +5,7 @@
|
||||
#include "nix/util/finally.hh"
|
||||
#include "nix/util/unix-domain-socket.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/util.hh"
|
||||
#include "nix/store/posix-fs-canonicalise.hh"
|
||||
|
||||
#include "store-config-private.hh"
|
||||
@@ -906,9 +907,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
#endif
|
||||
;
|
||||
|
||||
printInfo(
|
||||
"note: currently hard linking saves %.2f MiB",
|
||||
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
||||
printInfo("note: currently hard linking saves %s", renderSize(unsharedSize - actualSize - overhead));
|
||||
}
|
||||
|
||||
/* While we're at it, vacuum the database. */
|
||||
|
||||
@@ -150,7 +150,7 @@ std::vector<Path> getUserConfigFiles()
|
||||
return files;
|
||||
}
|
||||
|
||||
unsigned int Settings::getDefaultCores() const
|
||||
unsigned int Settings::getDefaultCores()
|
||||
{
|
||||
const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency());
|
||||
const unsigned int maxCPU = getMaxCPU();
|
||||
|
||||
@@ -51,224 +51,211 @@ std::string HttpBinaryCacheStoreConfig::doc()
|
||||
;
|
||||
}
|
||||
|
||||
class HttpBinaryCacheStore : public virtual BinaryCacheStore
|
||||
HttpBinaryCacheStore::HttpBinaryCacheStore(ref<Config> config)
|
||||
: Store{*config} // TODO it will actually mutate the configuration
|
||||
, BinaryCacheStore{*config}
|
||||
, config{config}
|
||||
{
|
||||
struct State
|
||||
{
|
||||
bool enabled = true;
|
||||
std::chrono::steady_clock::time_point disabledUntil;
|
||||
};
|
||||
diskCache = getNarInfoDiskCache();
|
||||
}
|
||||
|
||||
Sync<State> _state;
|
||||
|
||||
public:
|
||||
|
||||
using Config = HttpBinaryCacheStoreConfig;
|
||||
|
||||
ref<Config> config;
|
||||
|
||||
HttpBinaryCacheStore(ref<Config> config)
|
||||
: Store{*config} // TODO it will actually mutate the configuration
|
||||
, BinaryCacheStore{*config}
|
||||
, config{config}
|
||||
{
|
||||
diskCache = getNarInfoDiskCache();
|
||||
}
|
||||
|
||||
void init() override
|
||||
{
|
||||
// FIXME: do this lazily?
|
||||
// For consistent cache key handling, use the reference without parameters
|
||||
// This matches what's used in Store::queryPathInfo() lookups
|
||||
auto cacheKey = config->getReference().render(/*withParams=*/false);
|
||||
|
||||
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) {
|
||||
config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
|
||||
config->priority.setDefault(cacheInfo->priority);
|
||||
} else {
|
||||
try {
|
||||
BinaryCacheStore::init();
|
||||
} catch (UploadToHTTP &) {
|
||||
throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string());
|
||||
}
|
||||
diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority);
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
void maybeDisable()
|
||||
{
|
||||
auto state(_state.lock());
|
||||
if (state->enabled && settings.tryFallback) {
|
||||
int t = 60;
|
||||
printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t);
|
||||
state->enabled = false;
|
||||
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
|
||||
}
|
||||
}
|
||||
|
||||
void checkEnabled()
|
||||
{
|
||||
auto state(_state.lock());
|
||||
if (state->enabled)
|
||||
return;
|
||||
if (std::chrono::steady_clock::now() > state->disabledUntil) {
|
||||
state->enabled = true;
|
||||
debug("re-enabling binary cache '%s'", config->getHumanReadableURI());
|
||||
return;
|
||||
}
|
||||
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
|
||||
}
|
||||
|
||||
bool fileExists(const std::string & path) override
|
||||
{
|
||||
checkEnabled();
|
||||
void HttpBinaryCacheStore::init()
|
||||
{
|
||||
// FIXME: do this lazily?
|
||||
// For consistent cache key handling, use the reference without parameters
|
||||
// This matches what's used in Store::queryPathInfo() lookups
|
||||
auto cacheKey = config->getReference().render(/*withParams=*/false);
|
||||
|
||||
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) {
|
||||
config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
|
||||
config->priority.setDefault(cacheInfo->priority);
|
||||
} else {
|
||||
try {
|
||||
FileTransferRequest request(makeRequest(path));
|
||||
request.head = true;
|
||||
getFileTransfer()->download(request);
|
||||
return true;
|
||||
} catch (FileTransferError & e) {
|
||||
/* S3 buckets return 403 if a file doesn't exist and the
|
||||
bucket is unlistable, so treat 403 as 404. */
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
return false;
|
||||
maybeDisable();
|
||||
throw;
|
||||
BinaryCacheStore::init();
|
||||
} catch (UploadToHTTP &) {
|
||||
throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string());
|
||||
}
|
||||
diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority);
|
||||
}
|
||||
}
|
||||
|
||||
void upsertFile(
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType) override
|
||||
{
|
||||
auto req = makeRequest(path);
|
||||
|
||||
auto data = StreamToSourceAdapter(istream).drain();
|
||||
|
||||
// Determine compression method based on file type
|
||||
std::string compressionMethod;
|
||||
if (hasSuffix(path, ".narinfo"))
|
||||
compressionMethod = config->narinfoCompression;
|
||||
else if (hasSuffix(path, ".ls"))
|
||||
compressionMethod = config->lsCompression;
|
||||
else if (hasPrefix(path, "log/"))
|
||||
compressionMethod = config->logCompression;
|
||||
|
||||
// Apply compression if configured
|
||||
if (!compressionMethod.empty()) {
|
||||
data = compress(compressionMethod, data);
|
||||
req.headers.emplace_back("Content-Encoding", compressionMethod);
|
||||
}
|
||||
|
||||
req.data = std::move(data);
|
||||
req.mimeType = mimeType;
|
||||
|
||||
try {
|
||||
getFileTransfer()->upload(req);
|
||||
} catch (FileTransferError & e) {
|
||||
throw UploadToHTTP(
|
||||
"while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
|
||||
}
|
||||
}
|
||||
|
||||
FileTransferRequest makeRequest(const std::string & path)
|
||||
{
|
||||
/* Otherwise the last path fragment will get discarded. */
|
||||
auto cacheUriWithTrailingSlash = config->cacheUri;
|
||||
if (!cacheUriWithTrailingSlash.path.empty())
|
||||
cacheUriWithTrailingSlash.path.push_back("");
|
||||
|
||||
/* path is not a path, but a full relative or absolute
|
||||
URL, e.g. we've seen in the wild NARINFO files have a URL
|
||||
field which is
|
||||
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj`
|
||||
(note the query param) and that gets passed here. */
|
||||
auto result = parseURLRelative(path, cacheUriWithTrailingSlash);
|
||||
|
||||
/* For S3 URLs, preserve query parameters from the base URL when the
|
||||
relative path doesn't have its own query parameters. This is needed
|
||||
to preserve S3-specific parameters like endpoint and region. */
|
||||
if (config->cacheUri.scheme == "s3" && result.query.empty()) {
|
||||
result.query = config->cacheUri.query;
|
||||
}
|
||||
|
||||
return FileTransferRequest(result);
|
||||
}
|
||||
|
||||
void getFile(const std::string & path, Sink & sink) override
|
||||
{
|
||||
checkEnabled();
|
||||
auto request(makeRequest(path));
|
||||
try {
|
||||
getFileTransfer()->download(std::move(request), sink);
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
throw NoSuchBinaryCacheFile(
|
||||
"file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI());
|
||||
maybeDisable();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override
|
||||
{
|
||||
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
||||
|
||||
try {
|
||||
checkEnabled();
|
||||
|
||||
auto request(makeRequest(path));
|
||||
|
||||
getFileTransfer()->enqueueFileTransfer(
|
||||
request, {[callbackPtr, this](std::future<FileTransferResult> result) {
|
||||
try {
|
||||
(*callbackPtr)(std::move(result.get().data));
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
return (*callbackPtr)({});
|
||||
maybeDisable();
|
||||
callbackPtr->rethrow();
|
||||
} catch (...) {
|
||||
callbackPtr->rethrow();
|
||||
}
|
||||
}});
|
||||
|
||||
} catch (...) {
|
||||
callbackPtr->rethrow();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<std::string> getNixCacheInfo() override
|
||||
{
|
||||
try {
|
||||
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
|
||||
return result.data;
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound)
|
||||
return std::nullopt;
|
||||
maybeDisable();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This isn't actually necessary read only. We support "upsert" now, so we
|
||||
* have a notion of authentication via HTTP POST/PUT.
|
||||
*
|
||||
* For now, we conservatively say we don't know.
|
||||
*
|
||||
* \todo try to expose our HTTP authentication status.
|
||||
*/
|
||||
std::optional<TrustedFlag> isTrustedClient() override
|
||||
{
|
||||
std::optional<std::string> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
|
||||
{
|
||||
if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty())
|
||||
return config->narinfoCompression;
|
||||
else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty())
|
||||
return config->lsCompression;
|
||||
else if (hasPrefix(path, "log/") && !config->logCompression.get().empty())
|
||||
return config->logCompression;
|
||||
else
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
void HttpBinaryCacheStore::maybeDisable()
|
||||
{
|
||||
auto state(_state.lock());
|
||||
if (state->enabled && settings.tryFallback) {
|
||||
int t = 60;
|
||||
printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t);
|
||||
state->enabled = false;
|
||||
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
void HttpBinaryCacheStore::checkEnabled()
|
||||
{
|
||||
auto state(_state.lock());
|
||||
if (state->enabled)
|
||||
return;
|
||||
if (std::chrono::steady_clock::now() > state->disabledUntil) {
|
||||
state->enabled = true;
|
||||
debug("re-enabling binary cache '%s'", config->getHumanReadableURI());
|
||||
return;
|
||||
}
|
||||
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
|
||||
}
|
||||
|
||||
bool HttpBinaryCacheStore::fileExists(const std::string & path)
|
||||
{
|
||||
checkEnabled();
|
||||
|
||||
try {
|
||||
FileTransferRequest request(makeRequest(path));
|
||||
request.method = HttpMethod::HEAD;
|
||||
getFileTransfer()->download(request);
|
||||
return true;
|
||||
} catch (FileTransferError & e) {
|
||||
/* S3 buckets return 403 if a file doesn't exist and the
|
||||
bucket is unlistable, so treat 403 as 404. */
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
return false;
|
||||
maybeDisable();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void HttpBinaryCacheStore::upsertFile(
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint)
|
||||
{
|
||||
auto req = makeRequest(path);
|
||||
req.method = HttpMethod::PUT;
|
||||
auto data = StreamToSourceAdapter(istream).drain();
|
||||
|
||||
auto compressionMethod = getCompressionMethod(path);
|
||||
|
||||
if (compressionMethod) {
|
||||
data = compress(*compressionMethod, data);
|
||||
req.headers.emplace_back("Content-Encoding", *compressionMethod);
|
||||
}
|
||||
|
||||
req.data = std::move(data);
|
||||
req.mimeType = mimeType;
|
||||
|
||||
try {
|
||||
getFileTransfer()->upload(req);
|
||||
} catch (FileTransferError & e) {
|
||||
UploadToHTTP err(e.message());
|
||||
err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string());
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
|
||||
{
|
||||
/* Otherwise the last path fragment will get discarded. */
|
||||
auto cacheUriWithTrailingSlash = config->cacheUri;
|
||||
if (!cacheUriWithTrailingSlash.path.empty())
|
||||
cacheUriWithTrailingSlash.path.push_back("");
|
||||
|
||||
/* path is not a path, but a full relative or absolute
|
||||
URL, e.g. we've seen in the wild NARINFO files have a URL
|
||||
field which is
|
||||
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj`
|
||||
(note the query param) and that gets passed here. */
|
||||
auto result = parseURLRelative(path, cacheUriWithTrailingSlash);
|
||||
|
||||
/* For S3 URLs, preserve query parameters from the base URL when the
|
||||
relative path doesn't have its own query parameters. This is needed
|
||||
to preserve S3-specific parameters like endpoint and region. */
|
||||
if (config->cacheUri.scheme == "s3" && result.query.empty()) {
|
||||
result.query = config->cacheUri.query;
|
||||
}
|
||||
|
||||
return FileTransferRequest(result);
|
||||
}
|
||||
|
||||
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
||||
{
|
||||
checkEnabled();
|
||||
auto request(makeRequest(path));
|
||||
try {
|
||||
getFileTransfer()->download(std::move(request), sink);
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
|
||||
throw NoSuchBinaryCacheFile(
|
||||
"file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI());
|
||||
maybeDisable();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void HttpBinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
|
||||
{
|
||||
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
|
||||
|
||||
try {
|
||||
checkEnabled();
|
||||
|
||||
auto request(makeRequest(path));
|
||||
|
||||
getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future<FileTransferResult> result) {
|
||||
try {
|
||||
(*callbackPtr)(std::move(result.get().data));
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound
|
||||
|| e.error == FileTransfer::Forbidden)
|
||||
return (*callbackPtr)({});
|
||||
maybeDisable();
|
||||
callbackPtr->rethrow();
|
||||
} catch (...) {
|
||||
callbackPtr->rethrow();
|
||||
}
|
||||
}});
|
||||
|
||||
} catch (...) {
|
||||
callbackPtr->rethrow();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<std::string> HttpBinaryCacheStore::getNixCacheInfo()
|
||||
{
|
||||
try {
|
||||
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
|
||||
return result.data;
|
||||
} catch (FileTransferError & e) {
|
||||
if (e.error == FileTransfer::NotFound)
|
||||
return std::nullopt;
|
||||
maybeDisable();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This isn't actually necessary read only. We support "upsert" now, so we
|
||||
* have a notion of authentication via HTTP POST/PUT.
|
||||
*
|
||||
* For now, we conservatively say we don't know.
|
||||
*
|
||||
* \todo try to expose our HTTP authentication status.
|
||||
*/
|
||||
std::optional<TrustedFlag> HttpBinaryCacheStore::isTrustedClient()
|
||||
{
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
ref<Store> HttpBinaryCacheStore::Config::openStore() const
|
||||
{
|
||||
|
||||
@@ -101,13 +101,27 @@ public:
|
||||
virtual bool fileExists(const std::string & path) = 0;
|
||||
|
||||
virtual void upsertFile(
|
||||
const std::string & path, std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) = 0;
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint) = 0;
|
||||
|
||||
void upsertFile(
|
||||
const std::string & path,
|
||||
// FIXME: use std::string_view
|
||||
std::string && data,
|
||||
const std::string & mimeType);
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint);
|
||||
|
||||
void upsertFile(
|
||||
const std::string & path,
|
||||
// FIXME: use std::string_view
|
||||
std::string && data,
|
||||
const std::string & mimeType)
|
||||
{
|
||||
auto size = data.size();
|
||||
upsertFile(path, std::move(data), mimeType, size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dump the contents of the specified file to a sink.
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include "nix/store/path.hh"
|
||||
#include "nix/util/file-content-address.hh"
|
||||
#include "nix/util/variant-wrapper.hh"
|
||||
#include "nix/util/json-impls.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -308,4 +309,15 @@ struct ContentAddressWithReferences
|
||||
Hash getHash() const;
|
||||
};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<ContentAddressMethod> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<ContentAddress> : std::true_type
|
||||
{};
|
||||
|
||||
} // namespace nix
|
||||
|
||||
JSON_IMPL(nix::ContentAddressMethod)
|
||||
JSON_IMPL(nix::ContentAddress)
|
||||
@@ -83,6 +83,17 @@ extern FileTransferSettings fileTransferSettings;
|
||||
|
||||
extern const unsigned int RETRY_TIME_MS_DEFAULT;
|
||||
|
||||
/**
|
||||
* HTTP methods supported by FileTransfer.
|
||||
*/
|
||||
enum struct HttpMethod {
|
||||
GET,
|
||||
PUT,
|
||||
HEAD,
|
||||
POST,
|
||||
DELETE,
|
||||
};
|
||||
|
||||
/**
|
||||
* Username and optional password for HTTP basic authentication.
|
||||
* These are used with curl's CURLOPT_USERNAME and CURLOPT_PASSWORD options
|
||||
@@ -99,9 +110,7 @@ struct FileTransferRequest
|
||||
VerbatimURL uri;
|
||||
Headers headers;
|
||||
std::string expectedETag;
|
||||
bool verifyTLS = true;
|
||||
bool head = false;
|
||||
bool post = false;
|
||||
HttpMethod method = HttpMethod::GET;
|
||||
size_t tries = fileTransferSettings.tries;
|
||||
unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
|
||||
ActivityId parentAct;
|
||||
@@ -128,14 +137,31 @@ struct FileTransferRequest
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the verb root for logging purposes.
|
||||
* The returned string is intended to be concatenated with "ing" to form the gerund,
|
||||
* e.g., "download" + "ing" -> "downloading", "upload" + "ing" -> "uploading".
|
||||
*/
|
||||
std::string verb() const
|
||||
{
|
||||
return data ? "upload" : "download";
|
||||
switch (method) {
|
||||
case HttpMethod::HEAD:
|
||||
case HttpMethod::GET:
|
||||
return "download";
|
||||
case HttpMethod::PUT:
|
||||
case HttpMethod::POST:
|
||||
assert(data);
|
||||
return "upload";
|
||||
case HttpMethod::DELETE:
|
||||
return "delet";
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
void setupForS3();
|
||||
|
||||
private:
|
||||
friend struct curlFileTransfer;
|
||||
void setupForS3();
|
||||
#if NIX_WITH_AWS_AUTH
|
||||
std::optional<std::string> awsSigV4Provider;
|
||||
#endif
|
||||
@@ -202,6 +228,11 @@ struct FileTransfer
|
||||
*/
|
||||
FileTransferResult upload(const FileTransferRequest & request);
|
||||
|
||||
/**
|
||||
* Synchronously delete a resource.
|
||||
*/
|
||||
FileTransferResult deleteResource(const FileTransferRequest & request);
|
||||
|
||||
/**
|
||||
* Download a file, writing its data to a sink. The sink will be
|
||||
* invoked on the thread of the caller.
|
||||
|
||||
@@ -77,7 +77,7 @@ public:
|
||||
|
||||
Settings();
|
||||
|
||||
unsigned int getDefaultCores() const;
|
||||
static unsigned int getDefaultCores();
|
||||
|
||||
Path nixPrefix;
|
||||
|
||||
@@ -427,7 +427,7 @@ public:
|
||||
R"(
|
||||
If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available.
|
||||
|
||||
It means that remote build hosts fetches as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
|
||||
It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
|
||||
This can drastically reduce build times if the network connection between the local machine and the remote build host is slow.
|
||||
)"};
|
||||
|
||||
@@ -503,7 +503,7 @@ public:
|
||||
by the Nix account, its group should be the group specified here,
|
||||
and its mode should be `1775`.
|
||||
|
||||
If the build users group is empty, builds areperformed under
|
||||
If the build users group is empty, builds are performed under
|
||||
the uid of the Nix process (that is, the uid of the caller if
|
||||
`NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
|
||||
`NIX_REMOTE` is `daemon`). Obviously, this should not be used
|
||||
@@ -847,8 +847,8 @@ public:
|
||||
4. The path to the build's scratch directory. This directory
|
||||
exists only if the build was run with `--keep-failed`.
|
||||
|
||||
The stderr and stdout output from the diff hook isn't
|
||||
displayed to the user. Instead, it print to the nix-daemon's log.
|
||||
The stderr and stdout output from the diff hook isn't displayed
|
||||
to the user. Instead, it prints to the nix-daemon's log.
|
||||
|
||||
When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
|
||||
configuration file, and cannot be passed at the command line.
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/store/binary-cache-store.hh"
|
||||
#include "nix/store/filetransfer.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -46,4 +50,51 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
|
||||
StoreReference getReference() const override;
|
||||
};
|
||||
|
||||
class HttpBinaryCacheStore : public virtual BinaryCacheStore
|
||||
{
|
||||
struct State
|
||||
{
|
||||
bool enabled = true;
|
||||
std::chrono::steady_clock::time_point disabledUntil;
|
||||
};
|
||||
|
||||
Sync<State> _state;
|
||||
|
||||
public:
|
||||
|
||||
using Config = HttpBinaryCacheStoreConfig;
|
||||
|
||||
ref<Config> config;
|
||||
|
||||
HttpBinaryCacheStore(ref<Config> config);
|
||||
|
||||
void init() override;
|
||||
|
||||
protected:
|
||||
|
||||
std::optional<std::string> getCompressionMethod(const std::string & path);
|
||||
|
||||
void maybeDisable();
|
||||
|
||||
void checkEnabled();
|
||||
|
||||
bool fileExists(const std::string & path) override;
|
||||
|
||||
void upsertFile(
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint) override;
|
||||
|
||||
FileTransferRequest makeRequest(std::string_view path);
|
||||
|
||||
void getFile(const std::string & path, Sink & sink) override;
|
||||
|
||||
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override;
|
||||
|
||||
std::optional<std::string> getNixCacheInfo() override;
|
||||
|
||||
std::optional<TrustedFlag> isTrustedClient() override;
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
@@ -78,7 +78,6 @@ struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore
|
||||
|
||||
LocalFSStore(const Config & params);
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override;
|
||||
ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
|
||||
std::shared_ptr<SourceAccessor> getFSAccessor(const StorePath & path, bool requireValidPath = true) override;
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ struct StructuredAttrs
|
||||
{
|
||||
static constexpr std::string_view envVarName{"__json"};
|
||||
|
||||
nlohmann::json structuredAttrs;
|
||||
nlohmann::json::object_t structuredAttrs;
|
||||
|
||||
bool operator==(const StructuredAttrs &) const = default;
|
||||
|
||||
@@ -45,7 +45,7 @@ struct StructuredAttrs
|
||||
*/
|
||||
static void checkKeyNotInUse(const StringPairs & env);
|
||||
|
||||
nlohmann::json prepareStructuredAttrs(
|
||||
nlohmann::json::object_t prepareStructuredAttrs(
|
||||
Store & store,
|
||||
const DerivationOptions & drvOptions,
|
||||
const StorePathSet & inputPaths,
|
||||
@@ -62,7 +62,7 @@ struct StructuredAttrs
|
||||
* `prepareStructuredAttrs`, *not* the original `structuredAttrs`
|
||||
* field.
|
||||
*/
|
||||
static std::string writeShell(const nlohmann::json & prepared);
|
||||
static std::string writeShell(const nlohmann::json::object_t & prepared);
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
#include "nix/store/references.hh"
|
||||
#include "nix/store/path.hh"
|
||||
#include "nix/util/source-accessor.hh"
|
||||
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -21,4 +25,57 @@ public:
|
||||
StorePathSet getResultPaths();
|
||||
};
|
||||
|
||||
/**
|
||||
* Result of scanning a single file for references.
|
||||
*/
|
||||
struct FileRefScanResult
|
||||
{
|
||||
CanonPath filePath; ///< The file that was scanned
|
||||
StorePathSet foundRefs; ///< Which store paths were found in this file
|
||||
};
|
||||
|
||||
/**
|
||||
* Scan a store path tree and report which references appear in which files.
|
||||
*
|
||||
* This is like scanForReferences() but provides per-file granularity.
|
||||
* Useful for cycle detection and detailed dependency analysis like `nix why-depends --precise`.
|
||||
*
|
||||
* The function walks the tree using the provided accessor and streams each file's
|
||||
* contents through a RefScanSink to detect hash references. For each file that
|
||||
* contains at least one reference, a callback is invoked with the file path and
|
||||
* the set of references found.
|
||||
*
|
||||
* Note: This function only searches for the hash part of store paths (e.g.,
|
||||
* "dc04vv14dak1c1r48qa0m23vr9jy8sm0"), not the name part. A store path like
|
||||
* "/nix/store/dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo" will be detected if the
|
||||
* hash appears anywhere in the scanned content, regardless of the "-foo" suffix.
|
||||
*
|
||||
* @param accessor Source accessor to read the tree
|
||||
* @param rootPath Root path to scan
|
||||
* @param refs Set of store paths to search for
|
||||
* @param callback Called for each file that contains at least one reference
|
||||
*/
|
||||
void scanForReferencesDeep(
|
||||
SourceAccessor & accessor,
|
||||
const CanonPath & rootPath,
|
||||
const StorePathSet & refs,
|
||||
std::function<void(FileRefScanResult)> callback);
|
||||
|
||||
/**
|
||||
* Scan a store path tree and return which references appear in which files.
|
||||
*
|
||||
* This is a convenience wrapper around the callback-based scanForReferencesDeep()
|
||||
* that collects all results into a map for efficient lookups.
|
||||
*
|
||||
* Note: This function only searches for the hash part of store paths, not the name part.
|
||||
* See the callback-based overload for details.
|
||||
*
|
||||
* @param accessor Source accessor to read the tree
|
||||
* @param rootPath Root path to scan
|
||||
* @param refs Set of store paths to search for
|
||||
* @return Map from file paths to the set of references found in each file
|
||||
*/
|
||||
std::map<CanonPath, StorePathSet>
|
||||
scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs);
|
||||
|
||||
} // namespace nix
|
||||
@@ -21,8 +21,6 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
Nix uses the `default` profile.
|
||||
)"};
|
||||
|
||||
public:
|
||||
|
||||
const Setting<std::string> region{
|
||||
this,
|
||||
"us-east-1",
|
||||
@@ -63,6 +61,12 @@ public:
|
||||
> addressing instead of virtual host based addressing.
|
||||
)"};
|
||||
|
||||
/**
|
||||
* Set of settings that are part of the S3 URI itself.
|
||||
* These are needed for region specification and other S3-specific settings.
|
||||
*/
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint};
|
||||
|
||||
static const std::string name()
|
||||
{
|
||||
return "S3 Binary Cache Store";
|
||||
@@ -71,6 +75,10 @@ public:
|
||||
static StringSet uriSchemes();
|
||||
|
||||
static std::string doc();
|
||||
|
||||
std::string getHumanReadableURI() const override;
|
||||
|
||||
ref<Store> openStore() const override;
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
@@ -26,6 +26,7 @@ struct ParsedS3URL
|
||||
std::optional<std::string> profile;
|
||||
std::optional<std::string> region;
|
||||
std::optional<std::string> scheme;
|
||||
std::optional<std::string> versionId;
|
||||
/**
|
||||
* The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
|
||||
* or an authority (so an IP address or a registered name).
|
||||
|
||||
@@ -609,7 +609,7 @@ public:
|
||||
/**
|
||||
* Write a NAR dump of a store path.
|
||||
*/
|
||||
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
|
||||
virtual void narFromPath(const StorePath & path, Sink & sink);
|
||||
|
||||
/**
|
||||
* For each path, if it's a derivation, build it. Building a
|
||||
|
||||
@@ -91,7 +91,7 @@ struct StoreDirConfig
|
||||
std::pair<StorePath, Hash> computeStorePath(
|
||||
std::string_view name,
|
||||
const SourcePath & path,
|
||||
ContentAddressMethod method = FileIngestionMethod::NixArchive,
|
||||
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
|
||||
const StorePathSet & references = {},
|
||||
PathFilter & filter = defaultPathFilter) const;
|
||||
|
||||
@@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override
|
||||
{
|
||||
LocalFSStore::narFromPath(path, sink);
|
||||
Store::narFromPath(path, sink);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -13,6 +13,7 @@ namespace nix {
|
||||
/* Note: you generally shouldn't change the protocol version. Define a
|
||||
new `WorkerProto::Feature` instead. */
|
||||
#define PROTOCOL_VERSION (1 << 8 | 38)
|
||||
#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18)
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
|
||||
@@ -152,6 +153,7 @@ enum struct WorkerProto::Op : uint64_t {
|
||||
AddIndirectRoot = 12,
|
||||
SyncWithGC = 13,
|
||||
FindRoots = 14,
|
||||
// ExportPath = 16, // removed
|
||||
QueryDeriver = 18, // obsolete
|
||||
SetOptions = 19,
|
||||
CollectGarbage = 20,
|
||||
@@ -161,6 +163,7 @@ enum struct WorkerProto::Op : uint64_t {
|
||||
QueryFailedPaths = 24,
|
||||
ClearFailedPaths = 25,
|
||||
QueryPathInfo = 26,
|
||||
// ImportPaths = 27, // removed
|
||||
QueryDerivationOutputNames = 28, // obsolete
|
||||
QueryPathFromHashPart = 29,
|
||||
QuerySubstitutablePathInfos = 30,
|
||||
|
||||
@@ -56,7 +56,8 @@ protected:
|
||||
void upsertFile(
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType) override
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint) override
|
||||
{
|
||||
auto path2 = config->binaryCacheDir + "/" + path;
|
||||
static std::atomic<int> counter{0};
|
||||
|
||||
@@ -112,13 +112,6 @@ std::shared_ptr<SourceAccessor> LocalFSStore::getFSAccessor(const StorePath & pa
|
||||
return std::make_shared<PosixSourceAccessor>(std::move(absPath));
|
||||
}
|
||||
|
||||
void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
if (!isValidPath(path))
|
||||
throw Error("path '%s' is not valid", printStorePath(path));
|
||||
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
|
||||
}
|
||||
|
||||
const std::string LocalFSStore::drvsLogDir = "drvs";
|
||||
|
||||
std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path)
|
||||
|
||||
@@ -1048,15 +1048,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
|
||||
/* In case we are not interested in reading the NAR: discard it. */
|
||||
bool narRead = false;
|
||||
Finally cleanup = [&]() {
|
||||
if (!narRead) {
|
||||
NullFileSystemObjectSink sink;
|
||||
if (!narRead)
|
||||
try {
|
||||
parseDump(sink, source);
|
||||
source.skip(info.narSize);
|
||||
} catch (...) {
|
||||
// TODO: should Interrupted be handled here?
|
||||
ignoreExceptionInDestructor();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
addTempRoot(info.path);
|
||||
|
||||
@@ -159,17 +159,19 @@ NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path,
|
||||
UnkeyedValidPathInfo::fromJSON(store, json),
|
||||
}};
|
||||
|
||||
auto & obj = getObject(json);
|
||||
|
||||
if (json.contains("url"))
|
||||
res.url = getString(valueAt(json, "url"));
|
||||
res.url = getString(valueAt(obj, "url"));
|
||||
|
||||
if (json.contains("compression"))
|
||||
res.compression = getString(valueAt(json, "compression"));
|
||||
res.compression = getString(valueAt(obj, "compression"));
|
||||
|
||||
if (json.contains("downloadHash"))
|
||||
res.fileHash = Hash::parseAny(getString(valueAt(json, "downloadHash")), std::nullopt);
|
||||
res.fileHash = Hash::parseAny(getString(valueAt(obj, "downloadHash")), std::nullopt);
|
||||
|
||||
if (json.contains("downloadSize"))
|
||||
res.fileSize = getUnsigned(valueAt(json, "downloadSize"));
|
||||
res.fileSize = getUnsigned(valueAt(obj, "downloadSize"));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -312,7 +312,7 @@ void LocalStore::optimiseStore()
|
||||
|
||||
optimiseStore(stats);
|
||||
|
||||
printInfo("%s freed by hard-linking %d files", showBytes(stats.bytesFreed), stats.filesLinked);
|
||||
printInfo("%s freed by hard-linking %d files", renderSize(stats.bytesFreed), stats.filesLinked);
|
||||
}
|
||||
|
||||
void LocalStore::optimisePath(const Path & path, RepairFlag repair)
|
||||
|
||||
@@ -33,7 +33,8 @@ std::optional<StructuredAttrs> StructuredAttrs::tryExtract(StringPairs & env)
|
||||
|
||||
std::pair<std::string_view, std::string> StructuredAttrs::unparse() const
|
||||
{
|
||||
return {envVarName, structuredAttrs.dump()};
|
||||
// TODO don't copy the JSON object just to dump it.
|
||||
return {envVarName, static_cast<nlohmann::json>(structuredAttrs).dump()};
|
||||
}
|
||||
|
||||
void StructuredAttrs::checkKeyNotInUse(const StringPairs & env)
|
||||
@@ -97,7 +98,7 @@ static nlohmann::json pathInfoToJSON(Store & store, const StorePathSet & storePa
|
||||
return jsonList;
|
||||
}
|
||||
|
||||
nlohmann::json StructuredAttrs::prepareStructuredAttrs(
|
||||
nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
|
||||
Store & store,
|
||||
const DerivationOptions & drvOptions,
|
||||
const StorePathSet & inputPaths,
|
||||
@@ -120,7 +121,7 @@ nlohmann::json StructuredAttrs::prepareStructuredAttrs(
|
||||
return json;
|
||||
}
|
||||
|
||||
std::string StructuredAttrs::writeShell(const nlohmann::json & json)
|
||||
std::string StructuredAttrs::writeShell(const nlohmann::json::object_t & json)
|
||||
{
|
||||
|
||||
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
|
||||
@@ -144,7 +145,7 @@ std::string StructuredAttrs::writeShell(const nlohmann::json & json)
|
||||
|
||||
std::string jsonSh;
|
||||
|
||||
for (auto & [key, value] : json.items()) {
|
||||
for (auto & [key, value] : json) {
|
||||
|
||||
if (!std::regex_match(key, shVarName))
|
||||
continue;
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
#include "nix/store/path-references.hh"
|
||||
#include "nix/util/hash.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/source-accessor.hh"
|
||||
#include "nix/util/canon-path.hh"
|
||||
#include "nix/util/logging.hh"
|
||||
|
||||
#include <map>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -54,4 +58,90 @@ StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathS
|
||||
return refsSink.getResultPaths();
|
||||
}
|
||||
|
||||
void scanForReferencesDeep(
|
||||
SourceAccessor & accessor,
|
||||
const CanonPath & rootPath,
|
||||
const StorePathSet & refs,
|
||||
std::function<void(FileRefScanResult)> callback)
|
||||
{
|
||||
// Recursive tree walker
|
||||
auto walk = [&](this auto & self, const CanonPath & path) -> void {
|
||||
auto stat = accessor.lstat(path);
|
||||
|
||||
switch (stat.type) {
|
||||
case SourceAccessor::tRegular: {
|
||||
// Create a fresh sink for each file to independently detect references.
|
||||
// RefScanSink accumulates found hashes globally - once a hash is found,
|
||||
// it remains in the result set. If we reused the same sink across files,
|
||||
// we couldn't distinguish which files contain which references, as a hash
|
||||
// found in an earlier file wouldn't be reported when found in later files.
|
||||
PathRefScanSink sink = PathRefScanSink::fromPaths(refs);
|
||||
|
||||
// Scan this file by streaming its contents through the sink
|
||||
accessor.readFile(path, sink);
|
||||
|
||||
// Get the references found in this file
|
||||
auto foundRefs = sink.getResultPaths();
|
||||
|
||||
// Report if we found anything in this file
|
||||
if (!foundRefs.empty()) {
|
||||
debug("scanForReferencesDeep: found %d references in %s", foundRefs.size(), path.abs());
|
||||
callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case SourceAccessor::tDirectory: {
|
||||
// Recursively scan directory contents
|
||||
auto entries = accessor.readDirectory(path);
|
||||
for (const auto & [name, entryType] : entries) {
|
||||
self(path / name);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case SourceAccessor::tSymlink: {
|
||||
// Create a fresh sink for the symlink target (same reason as regular files)
|
||||
PathRefScanSink sink = PathRefScanSink::fromPaths(refs);
|
||||
|
||||
// Scan symlink target for references
|
||||
auto target = accessor.readLink(path);
|
||||
sink(std::string_view(target));
|
||||
|
||||
// Get the references found in this symlink target
|
||||
auto foundRefs = sink.getResultPaths();
|
||||
|
||||
if (!foundRefs.empty()) {
|
||||
debug("scanForReferencesDeep: found %d references in symlink %s", foundRefs.size(), path.abs());
|
||||
callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case SourceAccessor::tChar:
|
||||
case SourceAccessor::tBlock:
|
||||
case SourceAccessor::tSocket:
|
||||
case SourceAccessor::tFifo:
|
||||
case SourceAccessor::tUnknown:
|
||||
default:
|
||||
throw Error("file '%s' has an unsupported type", path.abs());
|
||||
}
|
||||
};
|
||||
|
||||
// Start the recursive walk from the root
|
||||
walk(rootPath);
|
||||
}
|
||||
|
||||
std::map<CanonPath, StorePathSet>
|
||||
scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs)
|
||||
{
|
||||
std::map<CanonPath, StorePathSet> results;
|
||||
|
||||
scanForReferencesDeep(accessor, rootPath, refs, [&](FileRefScanResult result) {
|
||||
results[std::move(result.filePath)] = std::move(result.foundRefs);
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -73,7 +73,7 @@ void RemoteStore::initConnection(Connection & conn)
|
||||
try {
|
||||
auto [protoVersion, features] =
|
||||
WorkerProto::BasicClientConnection::handshake(conn.to, tee, PROTOCOL_VERSION, WorkerProto::allFeatures);
|
||||
if (protoVersion < 256 + 18)
|
||||
if (protoVersion < MINIMUM_PROTOCOL_VERSION)
|
||||
throw Error("the Nix daemon version is too old");
|
||||
conn.protoVersion = protoVersion;
|
||||
conn.features = features;
|
||||
|
||||
@@ -226,7 +226,7 @@ void RestrictedStore::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
if (!goal.isAllowed(path))
|
||||
throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
|
||||
LocalFSStore::narFromPath(path, sink);
|
||||
Store::narFromPath(path, sink);
|
||||
}
|
||||
|
||||
void RestrictedStore::ensurePath(const StorePath & path)
|
||||
|
||||
@@ -1,12 +1,138 @@
|
||||
#include "nix/store/s3-binary-cache-store.hh"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
#include "nix/store/http-binary-cache-store.hh"
|
||||
#include "nix/store/store-registration.hh"
|
||||
|
||||
#include <cassert>
|
||||
#include <ranges>
|
||||
#include <regex>
|
||||
|
||||
namespace nix {
|
||||
|
||||
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
|
||||
{
|
||||
public:
|
||||
S3BinaryCacheStore(ref<S3BinaryCacheStoreConfig> config)
|
||||
: Store{*config}
|
||||
, BinaryCacheStore{*config}
|
||||
, HttpBinaryCacheStore{config}
|
||||
, s3Config{config}
|
||||
{
|
||||
}
|
||||
|
||||
void upsertFile(
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint) override;
|
||||
|
||||
private:
|
||||
ref<S3BinaryCacheStoreConfig> s3Config;
|
||||
|
||||
/**
|
||||
* Creates a multipart upload for large objects to S3.
|
||||
*
|
||||
* @see
|
||||
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax
|
||||
*/
|
||||
std::string createMultipartUpload(
|
||||
std::string_view key, std::string_view mimeType, std::optional<std::string_view> contentEncoding);
|
||||
|
||||
/**
|
||||
* Uploads a single part of a multipart upload
|
||||
*
|
||||
* @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html#API_UploadPart_RequestSyntax
|
||||
*
|
||||
* @returns the [ETag](https://en.wikipedia.org/wiki/HTTP_ETag)
|
||||
*/
|
||||
std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data);
|
||||
|
||||
/**
|
||||
* Abort a multipart upload
|
||||
*
|
||||
* @see
|
||||
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html#API_AbortMultipartUpload_RequestSyntax
|
||||
*/
|
||||
void abortMultipartUpload(std::string_view key, std::string_view uploadId);
|
||||
};
|
||||
|
||||
void S3BinaryCacheStore::upsertFile(
|
||||
const std::string & path,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType,
|
||||
uint64_t sizeHint)
|
||||
{
|
||||
HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint);
|
||||
}
|
||||
|
||||
std::string S3BinaryCacheStore::createMultipartUpload(
|
||||
std::string_view key, std::string_view mimeType, std::optional<std::string_view> contentEncoding)
|
||||
{
|
||||
auto req = makeRequest(key);
|
||||
|
||||
// setupForS3() converts s3:// to https:// but strips query parameters
|
||||
// So we call it first, then add our multipart parameters
|
||||
req.setupForS3();
|
||||
|
||||
auto url = req.uri.parsed();
|
||||
url.query["uploads"] = "";
|
||||
req.uri = VerbatimURL(url);
|
||||
|
||||
req.method = HttpMethod::POST;
|
||||
req.data = "";
|
||||
req.mimeType = mimeType;
|
||||
|
||||
if (contentEncoding) {
|
||||
req.headers.emplace_back("Content-Encoding", *contentEncoding);
|
||||
}
|
||||
|
||||
auto result = getFileTransfer()->enqueueFileTransfer(req).get();
|
||||
|
||||
std::regex uploadIdRegex("<UploadId>([^<]+)</UploadId>");
|
||||
std::smatch match;
|
||||
|
||||
if (std::regex_search(result.data, match, uploadIdRegex)) {
|
||||
return match[1];
|
||||
}
|
||||
|
||||
throw Error("S3 CreateMultipartUpload response missing <UploadId>");
|
||||
}
|
||||
|
||||
std::string
|
||||
S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data)
|
||||
{
|
||||
auto req = makeRequest(key);
|
||||
req.method = HttpMethod::PUT;
|
||||
req.setupForS3();
|
||||
|
||||
auto url = req.uri.parsed();
|
||||
url.query["partNumber"] = std::to_string(partNumber);
|
||||
url.query["uploadId"] = uploadId;
|
||||
req.uri = VerbatimURL(url);
|
||||
req.data = std::move(data);
|
||||
req.mimeType = "application/octet-stream";
|
||||
|
||||
auto result = getFileTransfer()->enqueueFileTransfer(req).get();
|
||||
|
||||
if (result.etag.empty()) {
|
||||
throw Error("S3 UploadPart response missing ETag for part %d", partNumber);
|
||||
}
|
||||
|
||||
return std::move(result.etag);
|
||||
}
|
||||
|
||||
void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId)
|
||||
{
|
||||
auto req = makeRequest(key);
|
||||
req.setupForS3();
|
||||
|
||||
auto url = req.uri.parsed();
|
||||
url.query["uploadId"] = uploadId;
|
||||
req.uri = VerbatimURL(url);
|
||||
req.method = HttpMethod::DELETE;
|
||||
|
||||
getFileTransfer()->enqueueFileTransfer(req).get();
|
||||
}
|
||||
|
||||
StringSet S3BinaryCacheStoreConfig::uriSchemes()
|
||||
{
|
||||
return {"s3"};
|
||||
@@ -17,19 +143,31 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
|
||||
: StoreConfig(params)
|
||||
, HttpBinaryCacheStoreConfig(scheme, _cacheUri, params)
|
||||
{
|
||||
// For S3 stores, preserve S3-specific query parameters as part of the URL
|
||||
// These are needed for region specification and other S3-specific settings
|
||||
assert(cacheUri.query.empty());
|
||||
assert(cacheUri.scheme == "s3");
|
||||
|
||||
// Only copy S3-specific parameters to the URL query
|
||||
static const std::set<std::string> s3Params = {"region", "endpoint", "profile", "scheme"};
|
||||
for (const auto & [key, value] : params) {
|
||||
if (s3Params.contains(key)) {
|
||||
auto s3Params =
|
||||
std::views::transform(s3UriSettings, [](const AbstractSetting * setting) { return setting->name; });
|
||||
if (std::ranges::contains(s3Params, key)) {
|
||||
cacheUri.query[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const
|
||||
{
|
||||
auto reference = getReference();
|
||||
reference.params = [&]() {
|
||||
Params relevantParams;
|
||||
for (auto & setting : s3UriSettings)
|
||||
if (setting->overridden)
|
||||
relevantParams.insert({setting->name, reference.params.at(setting->name)});
|
||||
return relevantParams;
|
||||
}();
|
||||
return reference.render();
|
||||
}
|
||||
|
||||
std::string S3BinaryCacheStoreConfig::doc()
|
||||
{
|
||||
return R"(
|
||||
@@ -39,6 +177,13 @@ std::string S3BinaryCacheStoreConfig::doc()
|
||||
)";
|
||||
}
|
||||
|
||||
ref<Store> S3BinaryCacheStoreConfig::openStore() const
|
||||
{
|
||||
auto sharedThis = std::const_pointer_cast<S3BinaryCacheStoreConfig>(
|
||||
std::static_pointer_cast<const S3BinaryCacheStoreConfig>(shared_from_this()));
|
||||
return make_ref<S3BinaryCacheStore>(ref{sharedThis});
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
|
||||
|
||||
} // namespace nix
|
||||
@@ -27,7 +27,8 @@ like the following to be accessible:
|
||||
"Sid": "AllowDirectReads",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:GetBucketLocation"
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": [
|
||||
@@ -51,7 +52,7 @@ Consult the documentation linked above for further details.
|
||||
|
||||
### Authenticated reads to your S3 binary cache
|
||||
|
||||
Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket.
|
||||
Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject`, `s3:GetBucketLocation`, and `s3:ListBucket` actions on all objects in the bucket.
|
||||
The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this.
|
||||
|
||||
### Authenticated writes to your S3-compatible binary cache
|
||||
|
||||
@@ -48,6 +48,7 @@ try {
|
||||
.profile = getOptionalParam("profile"),
|
||||
.region = getOptionalParam("region"),
|
||||
.scheme = getOptionalParam("scheme"),
|
||||
.versionId = getOptionalParam("versionId"),
|
||||
.endpoint = [&]() -> decltype(ParsedS3URL::endpoint) {
|
||||
if (!endpoint)
|
||||
return std::monostate();
|
||||
@@ -73,6 +74,12 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
auto regionStr = region.transform(toView).value_or("us-east-1");
|
||||
auto schemeStr = scheme.transform(toView).value_or("https");
|
||||
|
||||
// Build query parameters (e.g., versionId if present)
|
||||
StringMap queryParams;
|
||||
if (versionId) {
|
||||
queryParams["versionId"] = *versionId;
|
||||
}
|
||||
|
||||
// Handle endpoint configuration using std::visit
|
||||
return std::visit(
|
||||
overloaded{
|
||||
@@ -85,6 +92,7 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL::Authority & auth) {
|
||||
@@ -96,6 +104,7 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority = auth,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL & endpointUrl) {
|
||||
@@ -107,6 +116,7 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
.scheme = endpointUrl.scheme,
|
||||
.authority = endpointUrl.authority,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
},
|
||||
|
||||
@@ -143,7 +143,7 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore
|
||||
|
||||
void narFromPath(const StorePath & path, Sink & sink) override
|
||||
{
|
||||
return LocalFSStore::narFromPath(path, sink);
|
||||
return Store::narFromPath(path, sink);
|
||||
}
|
||||
|
||||
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
|
||||
|
||||
@@ -300,6 +300,13 @@ ValidPathInfo Store::addToStoreSlow(
|
||||
return info;
|
||||
}
|
||||
|
||||
void Store::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
auto accessor = requireStoreObjectAccessor(path);
|
||||
SourcePath sourcePath{accessor};
|
||||
dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive);
|
||||
}
|
||||
|
||||
StringSet Store::Config::getDefaultSystemFeatures()
|
||||
{
|
||||
auto res = settings.systemFeatures.get();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"algorithm": "blake3",
|
||||
"format": "base64",
|
||||
"hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA="
|
||||
"algorithm": "blake3",
|
||||
"format": "base64",
|
||||
"hash": "nnDuFEmWX7YtBJBAoe0G7Dd0MNpuwTFz58T//NKL6YA="
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"algorithm": "sha256",
|
||||
"format": "base64",
|
||||
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
|
||||
"algorithm": "sha256",
|
||||
"format": "base64",
|
||||
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
|
||||
}
|
||||
@@ -70,7 +70,7 @@ TEST(valueAt, simpleObject)
|
||||
|
||||
auto nested = R"({ "hello": { "world": "" } })"_json;
|
||||
|
||||
ASSERT_EQ(valueAt(valueAt(getObject(nested), "hello"), "world"), "");
|
||||
ASSERT_EQ(valueAt(getObject(valueAt(getObject(nested), "hello")), "world"), "");
|
||||
}
|
||||
|
||||
TEST(valueAt, missingKey)
|
||||
@@ -119,10 +119,12 @@ TEST(getArray, wrongAssertions)
|
||||
{
|
||||
auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json;
|
||||
|
||||
ASSERT_THROW(getArray(valueAt(json, "object")), Error);
|
||||
ASSERT_THROW(getArray(valueAt(json, "string")), Error);
|
||||
ASSERT_THROW(getArray(valueAt(json, "int")), Error);
|
||||
ASSERT_THROW(getArray(valueAt(json, "boolean")), Error);
|
||||
auto & obj = getObject(json);
|
||||
|
||||
ASSERT_THROW(getArray(valueAt(obj, "object")), Error);
|
||||
ASSERT_THROW(getArray(valueAt(obj, "string")), Error);
|
||||
ASSERT_THROW(getArray(valueAt(obj, "int")), Error);
|
||||
ASSERT_THROW(getArray(valueAt(obj, "boolean")), Error);
|
||||
}
|
||||
|
||||
TEST(getString, rightAssertions)
|
||||
@@ -136,10 +138,12 @@ TEST(getString, wrongAssertions)
|
||||
{
|
||||
auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json;
|
||||
|
||||
ASSERT_THROW(getString(valueAt(json, "object")), Error);
|
||||
ASSERT_THROW(getString(valueAt(json, "array")), Error);
|
||||
ASSERT_THROW(getString(valueAt(json, "int")), Error);
|
||||
ASSERT_THROW(getString(valueAt(json, "boolean")), Error);
|
||||
auto & obj = getObject(json);
|
||||
|
||||
ASSERT_THROW(getString(valueAt(obj, "object")), Error);
|
||||
ASSERT_THROW(getString(valueAt(obj, "array")), Error);
|
||||
ASSERT_THROW(getString(valueAt(obj, "int")), Error);
|
||||
ASSERT_THROW(getString(valueAt(obj, "boolean")), Error);
|
||||
}
|
||||
|
||||
TEST(getIntegralNumber, rightAssertions)
|
||||
@@ -156,18 +160,20 @@ TEST(getIntegralNumber, wrongAssertions)
|
||||
auto json =
|
||||
R"({ "object": {}, "array": [], "string": "", "int": 0, "signed": -256, "large": 128, "boolean": false })"_json;
|
||||
|
||||
ASSERT_THROW(getUnsigned(valueAt(json, "object")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(json, "array")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(json, "string")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(json, "boolean")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(json, "signed")), Error);
|
||||
auto & obj = getObject(json);
|
||||
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(json, "object")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(json, "array")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(json, "string")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(json, "boolean")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(json, "large")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(json, "signed")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(obj, "object")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(obj, "array")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(obj, "string")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(obj, "boolean")), Error);
|
||||
ASSERT_THROW(getUnsigned(valueAt(obj, "signed")), Error);
|
||||
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(obj, "object")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(obj, "array")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(obj, "string")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(obj, "boolean")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(obj, "large")), Error);
|
||||
ASSERT_THROW(getInteger<int8_t>(valueAt(obj, "signed")), Error);
|
||||
}
|
||||
|
||||
TEST(getBoolean, rightAssertions)
|
||||
@@ -181,24 +187,28 @@ TEST(getBoolean, wrongAssertions)
|
||||
{
|
||||
auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json;
|
||||
|
||||
ASSERT_THROW(getBoolean(valueAt(json, "object")), Error);
|
||||
ASSERT_THROW(getBoolean(valueAt(json, "array")), Error);
|
||||
ASSERT_THROW(getBoolean(valueAt(json, "string")), Error);
|
||||
ASSERT_THROW(getBoolean(valueAt(json, "int")), Error);
|
||||
auto & obj = getObject(json);
|
||||
|
||||
ASSERT_THROW(getBoolean(valueAt(obj, "object")), Error);
|
||||
ASSERT_THROW(getBoolean(valueAt(obj, "array")), Error);
|
||||
ASSERT_THROW(getBoolean(valueAt(obj, "string")), Error);
|
||||
ASSERT_THROW(getBoolean(valueAt(obj, "int")), Error);
|
||||
}
|
||||
|
||||
TEST(optionalValueAt, existing)
|
||||
{
|
||||
auto json = R"({ "string": "ssh-rsa" })"_json;
|
||||
|
||||
ASSERT_EQ(optionalValueAt(json, "string"), std::optional{"ssh-rsa"});
|
||||
auto * ptr = optionalValueAt(getObject(json), "string");
|
||||
ASSERT_TRUE(ptr);
|
||||
ASSERT_EQ(*ptr, R"("ssh-rsa")"_json);
|
||||
}
|
||||
|
||||
TEST(optionalValueAt, empty)
|
||||
{
|
||||
auto json = R"({})"_json;
|
||||
|
||||
ASSERT_EQ(optionalValueAt(json, "string"), std::nullopt);
|
||||
ASSERT_EQ(optionalValueAt(getObject(json), "string"), nullptr);
|
||||
}
|
||||
|
||||
TEST(getNullable, null)
|
||||
|
||||
@@ -158,6 +158,7 @@ TEST(renderSize, misc)
|
||||
ASSERT_EQ(renderSize(972, true), " 0.9 KiB");
|
||||
ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down
|
||||
ASSERT_EQ(renderSize(1024, true), " 1.0 KiB");
|
||||
ASSERT_EQ(renderSize(-1024, true), " -1.0 KiB");
|
||||
ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB");
|
||||
ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB");
|
||||
ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB");
|
||||
|
||||
Loaded 100 of 123 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user