Compare commits
102
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1c0f416cb | ||
|
|
8104858643 | ||
|
|
a6c1d5637a | ||
|
|
27006cc8a9 | ||
|
|
06f21596a0 | ||
|
|
9254fab407 | ||
|
|
994324feda | ||
|
|
1f739961e5 | ||
|
|
cccfa385e6 | ||
|
|
9d2100a165 | ||
|
|
4769f3c0b2 | ||
|
|
ab354dc8f6 | ||
|
|
188cb798ad | ||
|
|
1aa7ab0dcf | ||
|
|
208ed3c538 | ||
|
|
b6add8dcc6 | ||
|
|
79750a3ccc | ||
|
|
30cd9e43e1 | ||
|
|
0695630eb5 | ||
|
|
89dc57f6aa | ||
|
|
f274a7273a | ||
|
|
675656ffba | ||
|
|
a5edc2d921 | ||
|
|
2f092870e4 | ||
|
|
b39da9c0c2 | ||
|
|
f536b25367 | ||
|
|
017fae3f14 | ||
|
|
018d6462de | ||
|
|
4a5d960952 | ||
|
|
8cf8a9151a | ||
|
|
4060ec3a8c | ||
|
|
e0830681e2 | ||
|
|
9f2795e588 | ||
|
|
12cee327a0 | ||
|
|
3b73dcba39 | ||
|
|
dfad4b1403 | ||
|
|
5f69fd3e8d | ||
|
|
47416968d2 | ||
|
|
ce38abb697 | ||
|
|
a38fc659cc | ||
|
|
85bbfd4493 | ||
|
|
d5d7594029 | ||
|
|
1fc5648204 | ||
|
|
d7e0bcaa51 | ||
|
|
4227d24bc3 | ||
|
|
7720dad11f | ||
|
|
832b81761e | ||
|
|
1c63cf4001 | ||
|
|
df7542247e | ||
|
|
49f666c64d | ||
|
|
11f5a3124b | ||
|
|
92e698426b | ||
|
|
906334686c | ||
|
|
0ffe83aa14 | ||
|
|
8e044f1ed0 | ||
|
|
453dbab1e8 | ||
|
|
fc81840a8e | ||
|
|
71bdb33a36 | ||
|
|
0595c5f7ee | ||
|
|
11f108d898 | ||
|
|
128b2b5c56 | ||
|
|
508d4463e5 | ||
|
|
3c8e45c061 | ||
|
|
ec91479076 | ||
|
|
b398c14045 | ||
|
|
9a6f1e6266 | ||
|
|
1c728ce0de | ||
|
|
e145632aef | ||
|
|
5cdf2a19bd | ||
|
|
bb74677b08 | ||
|
|
3cfac9b079 | ||
|
|
198628790b | ||
|
|
54d2268d84 | ||
|
|
8c74aadbf7 | ||
|
|
3a62be7227 | ||
|
|
a6eb2e91b7 | ||
|
|
76c09bf3d4 | ||
|
|
de6fdb7da5 | ||
|
|
b54dfb66dd | ||
|
|
bb718d20a2 | ||
|
|
3b3bd018a5 | ||
|
|
26b86a02db | ||
|
|
8358409fd0 | ||
|
|
46670a7f46 | ||
|
|
ea96e6d07c | ||
|
|
7e3de5361a | ||
|
|
7b3d7eb634 | ||
|
|
819a61acae | ||
|
|
ccba158780 | ||
|
|
4945c38b88 | ||
|
|
0f18076f3a | ||
|
|
c6ddc5cf1d | ||
|
|
8b955d80c2 | ||
|
|
3e832b61ec | ||
|
|
fd6c4614cf | ||
|
|
99baaf7444 | ||
|
|
46895edfce | ||
|
|
17f07f6c04 | ||
|
|
9c2be01285 | ||
|
|
8493c541fa | ||
|
|
68a802d253 | ||
|
|
af6326dfa4 |
No files matched your search
@@ -26,7 +26,7 @@ jobs:
|
|||||||
# required to find all branches
|
# required to find all branches
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Create backport PRs
|
- name: Create backport PRs
|
||||||
uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
|
uses: korthout/backport-action@c656f5d5851037b2b38fb5db2691a03fa229e3b2 # v4.0.1
|
||||||
id: backport
|
id: backport
|
||||||
with:
|
with:
|
||||||
# Config README: https://github.com/korthout/backport-action#backport-action
|
# Config README: https://github.com/korthout/backport-action#backport-action
|
||||||
|
|||||||
@@ -125,13 +125,13 @@ jobs:
|
|||||||
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
|
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
|
||||||
if: ${{ matrix.instrumented }}
|
if: ${{ matrix.instrumented }}
|
||||||
- name: Upload coverage reports
|
- name: Upload coverage reports
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@v6
|
||||||
with:
|
with:
|
||||||
name: coverage-reports
|
name: coverage-reports
|
||||||
path: coverage-reports/
|
path: coverage-reports/
|
||||||
if: ${{ matrix.instrumented }}
|
if: ${{ matrix.instrumented }}
|
||||||
- name: Upload installer tarball
|
- name: Upload installer tarball
|
||||||
uses: actions/upload-artifact@v5
|
uses: actions/upload-artifact@v6
|
||||||
with:
|
with:
|
||||||
name: installer-${{matrix.os}}
|
name: installer-${{matrix.os}}
|
||||||
path: out/*
|
path: out/*
|
||||||
@@ -164,7 +164,7 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
- name: Download installer tarball
|
- name: Download installer tarball
|
||||||
uses: actions/download-artifact@v6
|
uses: actions/download-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: installer-${{matrix.os}}
|
name: installer-${{matrix.os}}
|
||||||
path: out
|
path: out
|
||||||
@@ -174,7 +174,7 @@ jobs:
|
|||||||
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
||||||
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
|
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
|
||||||
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
|
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
|
||||||
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
|
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
|
||||||
if: ${{ !matrix.experimental-installer }}
|
if: ${{ !matrix.experimental-installer }}
|
||||||
with:
|
with:
|
||||||
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ bash = find_program('bash', native : true)
|
|||||||
# HTML manual dependencies (conditional)
|
# HTML manual dependencies (conditional)
|
||||||
if get_option('html-manual')
|
if get_option('html-manual')
|
||||||
mdbook = find_program('mdbook', native : true)
|
mdbook = find_program('mdbook', native : true)
|
||||||
rsync = find_program('rsync', required : true, native : true)
|
|
||||||
endif
|
endif
|
||||||
|
|
||||||
pymod = import('python')
|
pymod = import('python')
|
||||||
@@ -126,7 +125,12 @@ if get_option('html-manual')
|
|||||||
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
|
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
|
||||||
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
|
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
|
||||||
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
|
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
|
||||||
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
|
# Copy source to build directory, excluding the build directory itself
|
||||||
|
# (which is present when built as an individual component).
|
||||||
|
# Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests).
|
||||||
|
(cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -)
|
||||||
|
chmod -R u+w @2@
|
||||||
|
find @2@ -name '*.drv' -delete
|
||||||
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
|
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
|
||||||
rm -rf @2@/manual
|
rm -rf @2@/manual
|
||||||
mv @2@/html @2@/manual
|
mv @2@/html @2@/manual
|
||||||
@@ -138,7 +142,6 @@ if get_option('html-manual')
|
|||||||
mdbook.full_path(),
|
mdbook.full_path(),
|
||||||
meson.current_build_dir(),
|
meson.current_build_dir(),
|
||||||
meson.project_version(),
|
meson.project_version(),
|
||||||
rsync.full_path(),
|
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
input : [
|
input : [
|
||||||
|
|||||||
@@ -10,7 +10,6 @@
|
|||||||
mdbook,
|
mdbook,
|
||||||
jq,
|
jq,
|
||||||
python3,
|
python3,
|
||||||
rsync,
|
|
||||||
nix-cli,
|
nix-cli,
|
||||||
changelog-d,
|
changelog-d,
|
||||||
json-schema-for-humans,
|
json-schema-for-humans,
|
||||||
@@ -54,6 +53,8 @@ mkMesonDerivation (finalAttrs: {
|
|||||||
../../src/libstore-tests/data/nar-info
|
../../src/libstore-tests/data/nar-info
|
||||||
../../src/libstore-tests/data/build-result
|
../../src/libstore-tests/data/build-result
|
||||||
../../src/libstore-tests/data/dummy-store
|
../../src/libstore-tests/data/dummy-store
|
||||||
|
# For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/
|
||||||
|
../../tests/functional/derivation
|
||||||
# Too many different types of files to filter for now
|
# Too many different types of files to filter for now
|
||||||
../../doc/manual
|
../../doc/manual
|
||||||
./.
|
./.
|
||||||
@@ -90,7 +91,6 @@ mkMesonDerivation (finalAttrs: {
|
|||||||
]
|
]
|
||||||
++ lib.optionals buildHtmlManual [
|
++ lib.optionals buildHtmlManual [
|
||||||
mdbook
|
mdbook
|
||||||
rsync
|
|
||||||
json-schema-for-humans
|
json-schema-for-humans
|
||||||
]
|
]
|
||||||
++ lib.optionals (!officialRelease && buildHtmlManual) [
|
++ lib.optionals (!officialRelease && buildHtmlManual) [
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: Channel URLs migrated to channels.nixos.org subdomain
|
|
||||||
prs: [14518]
|
|
||||||
issues: [14517]
|
|
||||||
---
|
|
||||||
|
|
||||||
Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix.
|
|
||||||
|
|
||||||
The subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: "JSON format changes for store path info and derivations"
|
|
||||||
prs: []
|
|
||||||
issues: []
|
|
||||||
---
|
|
||||||
|
|
||||||
JSON formats for store path info and derivations have been updated with new versions and structured fields.
|
|
||||||
|
|
||||||
## Store Path Info JSON
|
|
||||||
|
|
||||||
`nix path-info --json` now requires a `--json-format` flag to specify the output format version.
|
|
||||||
Using `--json` without `--json-format` is deprecated and will become an error in a future release.
|
|
||||||
For now, it defaults to version 1 with a warning, for a smoother migration.
|
|
||||||
|
|
||||||
### Version 1 (`--json-format 1`)
|
|
||||||
|
|
||||||
This is the legacy format, preserved for backwards compatibility:
|
|
||||||
|
|
||||||
- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
|
|
||||||
- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
|
|
||||||
- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
|
|
||||||
- Now includes `"storeDir"` field at the top level
|
|
||||||
|
|
||||||
### Version 2 (`--json-format 2`)
|
|
||||||
|
|
||||||
The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
|
|
||||||
|
|
||||||
- **Nested structure with top-level metadata**:
|
|
||||||
|
|
||||||
The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"version": 2,
|
|
||||||
"storeDir": "/nix/store",
|
|
||||||
"info": { ... }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The map from store bath base names to store object info is nested under the `info` field.
|
|
||||||
|
|
||||||
- **Store path base names instead of full paths**:
|
|
||||||
|
|
||||||
Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
|
|
||||||
Combined with `storeDir`, the full path can be reconstructed.
|
|
||||||
|
|
||||||
- **Structured `ca` field**:
|
|
||||||
|
|
||||||
Content address is now a structured JSON object instead of a string:
|
|
||||||
|
|
||||||
- Old: `"ca": "fixed:r:sha256:1abc..."`
|
|
||||||
- New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
|
|
||||||
- Still `null` values for input-addressed store objects
|
|
||||||
|
|
||||||
The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
|
|
||||||
|
|
||||||
Nix currently only produces, and doesn't consume this format.
|
|
||||||
|
|
||||||
Additionally the following field is added to both formats.
|
|
||||||
(The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.)
|
|
||||||
|
|
||||||
- **`version` field**:
|
|
||||||
|
|
||||||
All store path info JSON now includes `"version": <1|2>`.
|
|
||||||
|
|
||||||
- **`storeDir` field**:
|
|
||||||
|
|
||||||
Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
|
|
||||||
|
|
||||||
## Derivation JSON (Version 4)
|
|
||||||
|
|
||||||
The derivation JSON format has been updated from version 3 to version 4:
|
|
||||||
|
|
||||||
- **Restructured inputs**:
|
|
||||||
|
|
||||||
Inputs are now nested under an `inputs` object:
|
|
||||||
|
|
||||||
- Old: `"inputSrcs": [...], "inputDrvs": {...}`
|
|
||||||
- New: `"inputs": {"srcs": [...], "drvs": {...}}`
|
|
||||||
|
|
||||||
- **Consistent content addresses**:
|
|
||||||
|
|
||||||
Fixed content-addressed outputs now use structured JSON format.
|
|
||||||
This is the same format as `ca` in store path info (after the new version).
|
|
||||||
|
|
||||||
Version 3 and earlier formats are *not* accepted when reading.
|
|
||||||
|
|
||||||
**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
|
|
||||||
prs: [14614]
|
|
||||||
issues: [11728]
|
|
||||||
---
|
|
||||||
|
|
||||||
The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
|
|
||||||
[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
|
|
||||||
|
|
||||||
This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
|
|
||||||
|
|
||||||
Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: Interrupting REPL commands works more than once
|
|
||||||
issues: [13481]
|
|
||||||
---
|
|
||||||
|
|
||||||
Previously, this only worked once per REPL session; further attempts would be ignored.
|
|
||||||
This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
|
|
||||||
This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: "Improved S3 binary cache support via HTTP"
|
|
||||||
prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421]
|
|
||||||
issues: [13084, 12671, 11748, 12403]
|
|
||||||
---
|
|
||||||
|
|
||||||
S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native
|
|
||||||
AWS SigV4 authentication instead of the AWS C++ SDK, providing significant
|
|
||||||
improvements:
|
|
||||||
|
|
||||||
- **Reduced memory usage**: Eliminates memory buffering issues that caused
|
|
||||||
segfaults with large files
|
|
||||||
- **Fixed upload reliability**: Resolves AWS SDK chunking errors
|
|
||||||
(`InvalidChunkSizeError`)
|
|
||||||
- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full
|
|
||||||
`aws-cpp-sdk`, reducing build complexity
|
|
||||||
|
|
||||||
The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
|
|
||||||
management.
|
|
||||||
|
|
||||||
All existing S3 URL formats and parameters remain supported, however the store
|
|
||||||
settings for configuring multipart uploads have changed:
|
|
||||||
|
|
||||||
- **`multipart-upload`** (default: `false`): Enable multipart uploads for large
|
|
||||||
files. When enabled, files exceeding the multipart threshold will be uploaded
|
|
||||||
in multiple parts.
|
|
||||||
|
|
||||||
- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using
|
|
||||||
multipart uploads. Files smaller than this will use regular PUT requests.
|
|
||||||
Only takes effect when `multipart-upload` is enabled.
|
|
||||||
|
|
||||||
- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart
|
|
||||||
uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes
|
|
||||||
reduce the number of requests but use more memory.
|
|
||||||
|
|
||||||
- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
|
|
||||||
|
|
||||||
Note that this change also means Nix now supports S3 binary cache stores even
|
|
||||||
if built without `aws-crt-cpp`, but only for public buckets which do not
|
|
||||||
require authentication.
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: "S3 URLs now support object versioning via versionId parameter"
|
|
||||||
prs: [14274]
|
|
||||||
issues: [13955]
|
|
||||||
---
|
|
||||||
|
|
||||||
S3 URLs now support a `versionId` query parameter to fetch specific versions
|
|
||||||
of objects from S3 buckets with versioning enabled. This allows pinning to
|
|
||||||
exact object versions for reproducibility and protection against unexpected
|
|
||||||
changes:
|
|
||||||
|
|
||||||
```
|
|
||||||
s3://bucket/key?region=us-east-1&versionId=abc123def456
|
|
||||||
```
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
---
|
|
||||||
synopsis: "S3 binary cache stores now support storage class configuration"
|
|
||||||
prs: [14464]
|
|
||||||
issues: [7015]
|
|
||||||
---
|
|
||||||
|
|
||||||
S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
|
|
||||||
|
|
||||||
Example usage:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Use Glacier storage for long-term archival
|
|
||||||
nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
|
|
||||||
|
|
||||||
# Use Intelligent Tiering for automatic cost optimization
|
|
||||||
nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
|
|
||||||
```
|
|
||||||
|
|
||||||
The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
|
|
||||||
|
|
||||||
See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.
|
|
||||||
@@ -151,6 +151,7 @@
|
|||||||
- [Contributing](development/contributing.md)
|
- [Contributing](development/contributing.md)
|
||||||
- [Releases](release-notes/index.md)
|
- [Releases](release-notes/index.md)
|
||||||
{{#include ./SUMMARY-rl-next.md}}
|
{{#include ./SUMMARY-rl-next.md}}
|
||||||
|
- [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md)
|
||||||
- [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md)
|
- [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md)
|
||||||
- [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md)
|
- [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md)
|
||||||
- [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)
|
- [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)
|
||||||
|
|||||||
@@ -3,6 +3,10 @@
|
|||||||
This section provides some notes on how to start hacking on Nix.
|
This section provides some notes on how to start hacking on Nix.
|
||||||
To get the latest version of Nix from GitHub:
|
To get the latest version of Nix from GitHub:
|
||||||
|
|
||||||
|
> **Note**
|
||||||
|
>
|
||||||
|
> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo.
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ git clone https://github.com/NixOS/nix.git
|
$ git clone https://github.com/NixOS/nix.git
|
||||||
$ cd nix
|
$ cd nix
|
||||||
|
|||||||
@@ -6,14 +6,7 @@ Additionally, see [Testing Nix](./testing.md) for further instructions on how to
|
|||||||
|
|
||||||
## Building Nix with Debug Symbols
|
## Building Nix with Debug Symbols
|
||||||
|
|
||||||
In the development shell, set the `mesonBuildType` environment variable to `debug` before configuring the build:
|
In the development shell, `mesonBuildType` is set automatically to `debugoptimized`. This builds Nix with debug symbols, which are essential for effective debugging.
|
||||||
|
|
||||||
```console
|
|
||||||
[nix-shell]$ export mesonBuildType=debugoptimized
|
|
||||||
```
|
|
||||||
|
|
||||||
Then, proceed to build Nix as described in [Building Nix](./building.md).
|
|
||||||
This will build Nix with debug symbols, which are essential for effective debugging.
|
|
||||||
|
|
||||||
It is also possible to build without optimization for faster build:
|
It is also possible to build without optimization for faster build:
|
||||||
|
|
||||||
|
|||||||
@@ -1,27 +1,21 @@
|
|||||||
{{#include build-trace-entry-v1-fixed.md}}
|
{{#include build-trace-entry-v2-fixed.md}}
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
### Simple build trace entry
|
### Simple build trace entry
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{{#include schema/build-trace-entry-v1/simple.json}}
|
{{#include schema/build-trace-entry-v2/simple.json}}
|
||||||
```
|
|
||||||
|
|
||||||
### Build trace entry with dependencies
|
|
||||||
|
|
||||||
```json
|
|
||||||
{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Build trace entry with signature
|
### Build trace entry with signature
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{{#include schema/build-trace-entry-v1/with-signature.json}}
|
{{#include schema/build-trace-entry-v2/with-signature.json}}
|
||||||
```
|
```
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
## Raw Schema
|
## Raw Schema
|
||||||
|
|
||||||
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v1.json)
|
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json)
|
||||||
-->
|
-->
|
||||||
@@ -17,7 +17,7 @@ schemas = [
|
|||||||
'derivation-v4',
|
'derivation-v4',
|
||||||
'derivation-options-v1',
|
'derivation-options-v1',
|
||||||
'deriving-path-v1',
|
'deriving-path-v1',
|
||||||
'build-trace-entry-v1',
|
'build-trace-entry-v2',
|
||||||
'build-result-v1',
|
'build-result-v1',
|
||||||
'store-v1',
|
'store-v1',
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ properties:
|
|||||||
description: |
|
description: |
|
||||||
A mapping from output names to their build trace entries.
|
A mapping from output names to their build trace entries.
|
||||||
additionalProperties:
|
additionalProperties:
|
||||||
"$ref": "build-trace-entry-v1.yaml"
|
"$ref": "build-trace-entry-v2.yaml"
|
||||||
|
|
||||||
failure:
|
failure:
|
||||||
type: object
|
type: object
|
||||||
|
|||||||
File renamed without changes.
+13
-18
@@ -1,5 +1,5 @@
|
|||||||
"$schema": "http://json-schema.org/draft-04/schema"
|
"$schema": "http://json-schema.org/draft-04/schema"
|
||||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json"
|
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json"
|
||||||
title: Build Trace Entry
|
title: Build Trace Entry
|
||||||
description: |
|
description: |
|
||||||
A record of a successful build outcome for a specific derivation output.
|
A record of a successful build outcome for a specific derivation output.
|
||||||
@@ -11,10 +11,17 @@ description: |
|
|||||||
> This JSON format is currently
|
> This JSON format is currently
|
||||||
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
|
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
|
||||||
> and subject to change.
|
> and subject to change.
|
||||||
|
|
||||||
|
Verision history:
|
||||||
|
|
||||||
|
- Version 1: Original format
|
||||||
|
|
||||||
|
- Version 2: Remove `dependentRealisations`
|
||||||
|
|
||||||
|
type: object
|
||||||
required:
|
required:
|
||||||
- id
|
- id
|
||||||
- outPath
|
- outPath
|
||||||
- dependentRealisations
|
|
||||||
- signatures
|
- signatures
|
||||||
allOf:
|
allOf:
|
||||||
- "$ref": "#/$defs/key"
|
- "$ref": "#/$defs/key"
|
||||||
@@ -22,9 +29,11 @@ allOf:
|
|||||||
properties:
|
properties:
|
||||||
id: {}
|
id: {}
|
||||||
outPath: {}
|
outPath: {}
|
||||||
dependentRealisations: {}
|
|
||||||
signatures: {}
|
signatures: {}
|
||||||
additionalProperties: false
|
additionalProperties:
|
||||||
|
dependentRealisations:
|
||||||
|
description: deprecated field
|
||||||
|
type: object
|
||||||
|
|
||||||
"$defs":
|
"$defs":
|
||||||
key:
|
key:
|
||||||
@@ -60,7 +69,6 @@ additionalProperties: false
|
|||||||
type: object
|
type: object
|
||||||
required:
|
required:
|
||||||
- outPath
|
- outPath
|
||||||
- dependentRealisations
|
|
||||||
- signatures
|
- signatures
|
||||||
properties:
|
properties:
|
||||||
outPath:
|
outPath:
|
||||||
@@ -69,19 +77,6 @@ additionalProperties: false
|
|||||||
description: |
|
description: |
|
||||||
The path to the store object that resulted from building this derivation for the given output name.
|
The path to the store object that resulted from building this derivation for the given output name.
|
||||||
|
|
||||||
dependentRealisations:
|
|
||||||
type: object
|
|
||||||
title: Underlying Base Build Trace
|
|
||||||
description: |
|
|
||||||
This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
|
|
||||||
|
|
||||||
Keys are derivation output IDs (same format as the main `id` field).
|
|
||||||
Values are the store paths that those dependencies resolved to.
|
|
||||||
|
|
||||||
As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
|
|
||||||
This is the set of base build trace entries that this derived build trace is derived from.
|
|
||||||
(The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
|
|
||||||
|
|
||||||
patternProperties:
|
patternProperties:
|
||||||
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
|
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
|
||||||
"$ref": "store-path-v1.yaml"
|
"$ref": "store-path-v1.yaml"
|
||||||
@@ -70,7 +70,7 @@ properties:
|
|||||||
"^[A-Za-z0-9+/]{43}=$":
|
"^[A-Za-z0-9+/]{43}=$":
|
||||||
type: object
|
type: object
|
||||||
additionalProperties:
|
additionalProperties:
|
||||||
"$ref": "./build-trace-entry-v1.yaml#/$defs/value"
|
"$ref": "./build-trace-entry-v2.yaml#/$defs/value"
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
|
|
||||||
"$defs":
|
"$defs":
|
||||||
|
|||||||
@@ -0,0 +1,281 @@
|
|||||||
|
# Release 2.33.0 (2025-12-09)
|
||||||
|
|
||||||
|
## New features
|
||||||
|
|
||||||
|
- New command `nix registry resolve` [#14595](https://github.com/NixOS/nix/pull/14595)
|
||||||
|
|
||||||
|
This command looks up a flake registry input name and returns the flakeref it resolves to.
|
||||||
|
|
||||||
|
For example, looking up Nixpkgs:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ nix registry resolve nixpkgs
|
||||||
|
github:NixOS/nixpkgs/nixpkgs-unstable
|
||||||
|
```
|
||||||
|
|
||||||
|
Upstreamed from [Determinate Nix 3.14.0](https://github.com/DeterminateSystems/nix-src/pull/273).
|
||||||
|
|
||||||
|
- `nix flake clone` supports all input types [#14581](https://github.com/NixOS/nix/pull/14581)
|
||||||
|
|
||||||
|
`nix flake clone` now supports arbitrary input types. In particular, this allows you to clone tarball flakes, such as flakes on FlakeHub.
|
||||||
|
|
||||||
|
Upstreamed from [Determinate Nix 3.12.0](https://github.com/DeterminateSystems/nix-src/pull/229).
|
||||||
|
|
||||||
|
## Performance improvements
|
||||||
|
|
||||||
|
- Git fetcher computes `revCount`s using multiple threads [#14462](https://github.com/NixOS/nix/pull/14462)
|
||||||
|
|
||||||
|
When using Git repositories with a long history, calculating the `revCount` attribute can take a long time. Nix now computes `revCount` using multiple threads, making it much faster (e.g. 9.1s to 3.7s for Nixpkgs).
|
||||||
|
|
||||||
|
Note that if you don't need `revCount`, you can disable it altogether by setting the flake input attribute `shallow = true`.
|
||||||
|
|
||||||
|
Upstreamed from [Determinate Nix 3.12.2](https://github.com/DeterminateSystems/nix-src/pull/245).
|
||||||
|
|
||||||
|
- `builtins.stringLength` now runs in constant time [#14442](https://github.com/NixOS/nix/pull/14442)
|
||||||
|
|
||||||
|
The internal representation of strings has been replaced with a size-prefixed Pascal style string. Previously Nix stored strings as a NUL-terminated array of bytes, necessitating a linear scan to calculate the length.
|
||||||
|
|
||||||
|
- Uploads to `http://` and `https://` binary cache stores now run in constant memory [#14390](https://github.com/NixOS/nix/pull/14390)
|
||||||
|
|
||||||
|
Nix used to buffer the whole compressed NAR contents in memory. It now reads it in a streaming fashion.
|
||||||
|
|
||||||
|
- Channel URLs migrated to channels.nixos.org subdomain [#14517](https://github.com/NixOS/nix/issues/14517) [#14518](https://github.com/NixOS/nix/pull/14518)
|
||||||
|
|
||||||
|
Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix. This subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.
|
||||||
|
|
||||||
|
- Fix `download buffer is full; consider increasing the 'download-buffer-size' setting` warning [#11728](https://github.com/NixOS/nix/issues/11728) [#14614](https://github.com/NixOS/nix/pull/14614)
|
||||||
|
|
||||||
|
The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
|
||||||
|
[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
|
||||||
|
|
||||||
|
This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
|
||||||
|
|
||||||
|
Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
|
||||||
|
|
||||||
|
- Significantly improve tarball unpacking performance [#14689](https://github.com/NixOS/nix/pull/14689) [#14696](https://github.com/NixOS/nix/pull/14696) [#10683](https://github.com/NixOS/nix/issues/10683) [#11098](https://github.com/NixOS/nix/issues/11098)
|
||||||
|
|
||||||
|
Nix uses a content-addressed cache backed by libgit2 for deduplicating files fetched via `fetchTarball` and `github`, `tarball` flake inputs. Its usage has been significantly optimised to reduce the amount of I/O operations that are performed. For a typical nixpkgs source tarball this results in 200 times fewer system calls on Linux. In combination with libcurl pausing this alleviates performance regressions stemming from the tarball cache.
|
||||||
|
|
||||||
|
- Already valid derivations are no longer copied to the store [#14219](https://github.com/NixOS/nix/pull/14219)
|
||||||
|
|
||||||
|
This results in a modest speedup when using the Nix daemon.
|
||||||
|
|
||||||
|
- `nix nar ls` and `nix nar cat` are significantly faster and no longer buffer the whole NAR in memory [#14273](https://github.com/NixOS/nix/pull/14273) [#14732](https://github.com/NixOS/nix/pull/14732)
|
||||||
|
|
||||||
|
## S3 improvements
|
||||||
|
|
||||||
|
- Improved S3 binary cache support via HTTP [#11748](https://github.com/NixOS/nix/issues/11748) [#12403](https://github.com/NixOS/nix/issues/12403) [#12671](https://github.com/NixOS/nix/issues/12671) [#13084](https://github.com/NixOS/nix/issues/13084) [#13752](https://github.com/NixOS/nix/pull/13752) [#13823](https://github.com/NixOS/nix/pull/13823) [#14026](https://github.com/NixOS/nix/pull/14026) [#14120](https://github.com/NixOS/nix/pull/14120) [#14131](https://github.com/NixOS/nix/pull/14131) [#14135](https://github.com/NixOS/nix/pull/14135) [#14144](https://github.com/NixOS/nix/pull/14144) [#14170](https://github.com/NixOS/nix/pull/14170) [#14190](https://github.com/NixOS/nix/pull/14190) [#14198](https://github.com/NixOS/nix/pull/14198) [#14206](https://github.com/NixOS/nix/pull/14206) [#14209](https://github.com/NixOS/nix/pull/14209) [#14222](https://github.com/NixOS/nix/pull/14222) [#14223](https://github.com/NixOS/nix/pull/14223) [#14330](https://github.com/NixOS/nix/pull/14330) [#14333](https://github.com/NixOS/nix/pull/14333) [#14335](https://github.com/NixOS/nix/pull/14335) [#14336](https://github.com/NixOS/nix/pull/14336) [#14337](https://github.com/NixOS/nix/pull/14337) [#14350](https://github.com/NixOS/nix/pull/14350) [#14356](https://github.com/NixOS/nix/pull/14356) [#14357](https://github.com/NixOS/nix/pull/14357) [#14374](https://github.com/NixOS/nix/pull/14374) [#14375](https://github.com/NixOS/nix/pull/14375) [#14376](https://github.com/NixOS/nix/pull/14376) [#14377](https://github.com/NixOS/nix/pull/14377) [#14391](https://github.com/NixOS/nix/pull/14391) [#14393](https://github.com/NixOS/nix/pull/14393) [#14420](https://github.com/NixOS/nix/pull/14420) [#14421](https://github.com/NixOS/nix/pull/14421)
|
||||||
|
|
||||||
|
S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native AWS SigV4 authentication instead of the AWS C++ SDK, providing significant improvements:
|
||||||
|
|
||||||
|
- **Reduced memory usage**: Eliminates memory buffering issues that caused segfaults with large files
|
||||||
|
- **Fixed upload reliability**: Resolves AWS SDK chunking errors (`InvalidChunkSizeError`)
|
||||||
|
- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full `aws-cpp-sdk`, reducing build complexity
|
||||||
|
|
||||||
|
The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential management.
|
||||||
|
|
||||||
|
All existing S3 URL formats and parameters remain supported, however the store settings for configuring multipart uploads have changed:
|
||||||
|
|
||||||
|
- **`multipart-upload`** (default: `false`): Enable multipart uploads for large files. When enabled, files exceeding the multipart threshold will be uploaded in multiple parts.
|
||||||
|
|
||||||
|
- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using multipart uploads. Files smaller than this will use regular PUT requests. Only takes effect when `multipart-upload` is enabled.
|
||||||
|
|
||||||
|
- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the number of requests but use more memory.
|
||||||
|
|
||||||
|
- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
|
||||||
|
|
||||||
|
Note that this change also means Nix now supports S3 binary cache stores even if built without `aws-crt-cpp`, but only for public buckets which do not require authentication.
|
||||||
|
|
||||||
|
- S3 URLs now support object versioning via `versionId` parameter [#13955](https://github.com/NixOS/nix/issues/13955) [#14274](https://github.com/NixOS/nix/pull/14274)
|
||||||
|
|
||||||
|
S3 URLs now support a `versionId` query parameter to fetch specific versions
|
||||||
|
of objects from S3 buckets with versioning enabled. This allows pinning to
|
||||||
|
exact object versions for reproducibility and protection against unexpected
|
||||||
|
changes:
|
||||||
|
|
||||||
|
```
|
||||||
|
s3://bucket/key?region=us-east-1&versionId=abc123def456
|
||||||
|
```
|
||||||
|
|
||||||
|
- S3 binary cache stores now support storage class configuration [#7015](https://github.com/NixOS/nix/issues/7015) [#14464](https://github.com/NixOS/nix/pull/14464)
|
||||||
|
|
||||||
|
S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
|
||||||
|
|
||||||
|
Example usage:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Use Glacier storage for long-term archival
|
||||||
|
nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
|
||||||
|
|
||||||
|
# Use Intelligent Tiering for automatic cost optimization
|
||||||
|
nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
|
||||||
|
```
|
||||||
|
|
||||||
|
The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
|
||||||
|
|
||||||
|
See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.
|
||||||
|
|
||||||
|
|
||||||
|
## Store path info JSON format changes
|
||||||
|
|
||||||
|
The JSON format emitted by `nix path-info --json` has been updated to a new version with improved structure.
|
||||||
|
|
||||||
|
To maintain compatibility, `nix path-info --json` now requires a `--json-format` flag to specify the output format version.
|
||||||
|
Using `--json` without `--json-format` is deprecated and will become an error in a future release.
|
||||||
|
For now, it defaults to version 1 with a warning, for a smoother migration.
|
||||||
|
|
||||||
|
### Version 1 (`--json-format 1`)
|
||||||
|
|
||||||
|
This is the legacy format, preserved for backwards compatibility:
|
||||||
|
|
||||||
|
- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
|
||||||
|
- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
|
||||||
|
- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
|
||||||
|
- Now includes `"storeDir"` field at the top level
|
||||||
|
|
||||||
|
### Version 2 (`--json-format 2`)
|
||||||
|
|
||||||
|
The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
|
||||||
|
|
||||||
|
- **Nested structure with top-level metadata**:
|
||||||
|
|
||||||
|
The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"version": 2,
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"info": { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The map from store path base names to store object info is nested under the `info` field.
|
||||||
|
|
||||||
|
- **Store path base names instead of full paths**:
|
||||||
|
|
||||||
|
Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
|
||||||
|
Combined with `storeDir`, the full path can be reconstructed.
|
||||||
|
|
||||||
|
- **Structured `ca` field**:
|
||||||
|
|
||||||
|
Content address is now a structured JSON object instead of a string:
|
||||||
|
|
||||||
|
- Old: `"ca": "fixed:r:sha256:1abc..."`
|
||||||
|
- New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
|
||||||
|
- Still `null` values for input-addressed store objects
|
||||||
|
|
||||||
|
The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
|
||||||
|
|
||||||
|
Additionally the following fields are added to both formats:
|
||||||
|
|
||||||
|
- **`version` field**:
|
||||||
|
|
||||||
|
All store path info JSON now includes `"version": <1|2>`. The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.
|
||||||
|
|
||||||
|
- **`storeDir` field**:
|
||||||
|
|
||||||
|
Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
|
||||||
|
|
||||||
|
## Derivation JSON format changes
|
||||||
|
|
||||||
|
The derivation JSON format has been updated from version 3 to version 4:
|
||||||
|
|
||||||
|
- **Nested structure with top-level metadata**:
|
||||||
|
|
||||||
|
The output of `nix derivation show` is now wrapped in an object with `version` and `derivations` fields:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"version": 4,
|
||||||
|
"derivations": { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The map from derivation paths to derivation info is nested under the `derivations` field.
|
||||||
|
|
||||||
|
This matches the structure used for `nix path-info --json --json-format 2`, and likewise brings this command into compliance with the JSON guidelines.
|
||||||
|
|
||||||
|
- **Restructured inputs**:
|
||||||
|
|
||||||
|
Inputs are now nested under an `inputs` object:
|
||||||
|
|
||||||
|
- Old: `"inputSrcs": [...], "inputDrvs": {...}`
|
||||||
|
- New: `"inputs": {"srcs": [...], "drvs": {...}}`
|
||||||
|
|
||||||
|
- **Consistent content addresses**:
|
||||||
|
|
||||||
|
Fixed content-addressed outputs now use structured JSON format.
|
||||||
|
This is the same format as `ca` in store path info (after the new version).
|
||||||
|
|
||||||
|
Version 3 and earlier formats are *not* accepted when reading.
|
||||||
|
|
||||||
|
**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.
|
||||||
|
|
||||||
|
## Miscellaneous changes
|
||||||
|
|
||||||
|
- Git fetcher: Restore progress indication [#14487](https://github.com/NixOS/nix/pull/14487)
|
||||||
|
|
||||||
|
Nix used to feel "stuck" while it was cloning large repositories. Nix now shows Git's native progress indicator while fetching.
|
||||||
|
|
||||||
|
Upstreamed from [Determinate Nix 3.13.0](https://github.com/DeterminateSystems/nix-src/pull/250).
|
||||||
|
|
||||||
|
- Interrupting REPL commands works more than once [#13481](https://github.com/NixOS/nix/issues/13481)
|
||||||
|
|
||||||
|
Previously, this only worked once per REPL session; further attempts would be ignored.
|
||||||
|
This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
|
||||||
|
This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).
|
||||||
|
|
||||||
|
- NAR unpacking code has been rewritten to make use of dirfd-based `openat` and `openat2` system calls when available [#14597](https://github.com/NixOS/nix/pull/14597)
|
||||||
|
|
||||||
|
- Dynamic size unit rendering [#14423](https://github.com/NixOS/nix/pull/14423) [#14364](https://github.com/NixOS/nix/pull/14364)
|
||||||
|
|
||||||
|
Various commands and the progress bar now use dynamically determined size units instead
|
||||||
|
of always using `MiB`. For example, the progress bar now reports download status like:
|
||||||
|
|
||||||
|
```
|
||||||
|
[1/196/197 copied (773.7 MiB/2.1 GiB), 172.4/421.5 MiB DL]
|
||||||
|
```
|
||||||
|
|
||||||
|
Instead of:
|
||||||
|
|
||||||
|
```
|
||||||
|
[1/196/197 copied (773.7/2147.3 MiB), 172.4/421.5 MiB DL]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Contributors
|
||||||
|
|
||||||
|
This release was made possible by the following 33 contributors:
|
||||||
|
|
||||||
|
- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and)
|
||||||
|
- jonhermansen [**(@jonhermansen)**](https://github.com/jonhermansen)
|
||||||
|
- Arnout Engelen [**(@raboof)**](https://github.com/raboof)
|
||||||
|
- Jean-François Roche [**(@jfroche)**](https://github.com/jfroche)
|
||||||
|
- tomberek [**(@tomberek)**](https://github.com/tomberek)
|
||||||
|
- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra)
|
||||||
|
- Marcel [**(@MarcelCoding)**](https://github.com/MarcelCoding)
|
||||||
|
- David McFarland [**(@corngood)**](https://github.com/corngood)
|
||||||
|
- Soumyadip Sarkar [**(@neuralsorcerer)**](https://github.com/neuralsorcerer)
|
||||||
|
- Cole Helbling [**(@cole-h)**](https://github.com/cole-h)
|
||||||
|
- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314)
|
||||||
|
- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy)
|
||||||
|
- Alex Auvolat [**(@Alexis211)**](https://github.com/Alexis211)
|
||||||
|
- edef [**(@edef1c)**](https://github.com/edef1c)
|
||||||
|
- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium)
|
||||||
|
- Vinayak Goyal [**(@vinayakankugoyal)**](https://github.com/vinayakankugoyal)
|
||||||
|
- Graham Dennis [**(@GrahamDennis)**](https://github.com/GrahamDennis)
|
||||||
|
- Aspen Smith [**(@glittershark)**](https://github.com/glittershark)
|
||||||
|
- Jens Petersen [**(@juhp)**](https://github.com/juhp)
|
||||||
|
- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault)
|
||||||
|
- Peter Bynum [**(@pkpbynum)**](https://github.com/pkpbynum)
|
||||||
|
- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92)
|
||||||
|
- Alex Decious [**(@adeci)**](https://github.com/adeci)
|
||||||
|
- Matthieu Coudron [**(@teto)**](https://github.com/teto)
|
||||||
|
- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar)
|
||||||
|
- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii)
|
||||||
|
- Seth Flynn [**(@getchoo)**](https://github.com/getchoo)
|
||||||
|
- Robert Hensing [**(@roberth)**](https://github.com/roberth)
|
||||||
|
- Vladimir Panteleev [**(@CyberShadow)**](https://github.com/CyberShadow)
|
||||||
|
- bryango [**(@bryango)**](https://github.com/bryango)
|
||||||
|
- Henry [**(@cootshk)**](https://github.com/cootshk)
|
||||||
|
- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
|
||||||
|
- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)
|
||||||
@@ -224,5 +224,25 @@
|
|||||||
"42688647+netadr@users.noreply.github.com": "netadr",
|
"42688647+netadr@users.noreply.github.com": "netadr",
|
||||||
"matej.urbas@gmail.com": "urbas",
|
"matej.urbas@gmail.com": "urbas",
|
||||||
"ethanalexevans@gmail.com": "ethanavatar",
|
"ethanalexevans@gmail.com": "ethanavatar",
|
||||||
"greg.marti@gmail.com": "gmarti"
|
"greg.marti@gmail.com": "gmarti",
|
||||||
|
"arnout@bzzt.net": "raboof",
|
||||||
|
"vinayakankugoyal@gmail.com": "vinayakankugoyal",
|
||||||
|
"Radvendii@users.noreply.github.com": "Radvendii",
|
||||||
|
"jon@jh86.org": "jonhermansen",
|
||||||
|
"edef@edef.eu": "edef1c",
|
||||||
|
"pkpbynum@gmail.com": "pkpbynum",
|
||||||
|
"886074+teto@users.noreply.github.com": "teto",
|
||||||
|
"alex@adnab.me": "Alexis211",
|
||||||
|
"root@gws.fyi": "glittershark",
|
||||||
|
"me@m4rc3l.de": "MarcelCoding",
|
||||||
|
"taeer.bar-yam@bevuta.com": "Radvendii",
|
||||||
|
"martin.joerg@gmail.com": "mjoerg",
|
||||||
|
"git@cy.md": "CyberShadow",
|
||||||
|
"cootshk@duck.com": "cootshk",
|
||||||
|
"adam@dinwoodie.org": "me-and",
|
||||||
|
"domen@cachix.org": "domenkozar",
|
||||||
|
"alex.decious@gmail.com": "adeci",
|
||||||
|
"soumya.papanvk18@gmail.com": "neuralsorcerer",
|
||||||
|
"gdennis@anduril.com": null,
|
||||||
|
"graham.dennis@gmail.com": "GrahamDennis"
|
||||||
}
|
}
|
||||||
@@ -196,5 +196,21 @@
|
|||||||
"gmarti": "Gr\u00e9gory Marti",
|
"gmarti": "Gr\u00e9gory Marti",
|
||||||
"lovesegfault": "Bernardo Meurer",
|
"lovesegfault": "Bernardo Meurer",
|
||||||
"EphraimSiegfried": "Ephraim Siegfried",
|
"EphraimSiegfried": "Ephraim Siegfried",
|
||||||
"hgl": "Glen Huang"
|
"hgl": "Glen Huang",
|
||||||
|
"mjoerg": "Martin Joerg",
|
||||||
|
"Alexis211": "Alex Auvolat",
|
||||||
|
"domenkozar": "Domen Ko\u017ear",
|
||||||
|
"edef1c": "edef",
|
||||||
|
"cootshk": "Henry",
|
||||||
|
"raboof": "Arnout Engelen",
|
||||||
|
"pkpbynum": "Peter Bynum",
|
||||||
|
"glittershark": "Aspen Smith",
|
||||||
|
"MarcelCoding": "Marcel",
|
||||||
|
"teto": "Matthieu Coudron",
|
||||||
|
"jonhermansen": null,
|
||||||
|
"neuralsorcerer": "Soumyadip Sarkar",
|
||||||
|
"adeci": "Alex Decious",
|
||||||
|
"vinayakankugoyal": "Vinayak Goyal",
|
||||||
|
"me-and": "Adam Dinwoodie",
|
||||||
|
"GrahamDennis": "Graham Dennis"
|
||||||
}
|
}
|
||||||
+55
-39
@@ -148,6 +148,15 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
|||||||
isInternal =
|
isInternal =
|
||||||
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
|
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
|
||||||
|
|
||||||
|
activeComponentNames = lib.listToAttrs (
|
||||||
|
map (c: {
|
||||||
|
name = c.pname or c.name;
|
||||||
|
value = null;
|
||||||
|
}) activeComponents
|
||||||
|
);
|
||||||
|
|
||||||
|
isActiveComponent = name: activeComponentNames ? ${name};
|
||||||
|
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
pname = "shell-for-nix";
|
pname = "shell-for-nix";
|
||||||
@@ -190,27 +199,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
small =
|
small = finalAttrs.finalPackage.withActiveComponents (
|
||||||
(finalAttrs.finalPackage.withActiveComponents (
|
c:
|
||||||
c:
|
lib.intersectAttrs (lib.genAttrs [
|
||||||
lib.intersectAttrs (lib.genAttrs [
|
"nix-cli"
|
||||||
"nix-cli"
|
"nix-util-tests"
|
||||||
"nix-util-tests"
|
"nix-store-tests"
|
||||||
"nix-store-tests"
|
"nix-expr-tests"
|
||||||
"nix-expr-tests"
|
"nix-fetchers-tests"
|
||||||
"nix-fetchers-tests"
|
"nix-flake-tests"
|
||||||
"nix-flake-tests"
|
"nix-functional-tests"
|
||||||
"nix-functional-tests"
|
"nix-perl-bindings"
|
||||||
"nix-perl-bindings"
|
] (_: null)) c
|
||||||
] (_: null)) c
|
);
|
||||||
)).overrideAttrs
|
|
||||||
(o: {
|
|
||||||
mesonFlags = o.mesonFlags ++ [
|
|
||||||
# TODO: infer from activeComponents or vice versa
|
|
||||||
"-Dkaitai-struct-checks=false"
|
|
||||||
"-Djson-schema-checks=false"
|
|
||||||
];
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
|
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
|
||||||
@@ -258,10 +259,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
|||||||
|
|
||||||
# We use this shell with the local checkout, not unpackPhase.
|
# We use this shell with the local checkout, not unpackPhase.
|
||||||
src = null;
|
src = null;
|
||||||
|
|
||||||
# Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html
|
# Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html
|
||||||
# Remove when gdb fix is rolled out everywhere.
|
# Remove when gdb fix is rolled out everywhere.
|
||||||
separateDebugInfo = false;
|
separateDebugInfo = false;
|
||||||
|
|
||||||
|
mesonBuildType = "debugoptimized";
|
||||||
|
|
||||||
env = {
|
env = {
|
||||||
# For `make format`, to work without installing pre-commit
|
# For `make format`, to work without installing pre-commit
|
||||||
_NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml"
|
_NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml"
|
||||||
@@ -275,21 +279,33 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
|||||||
|
|
||||||
dontUseCmakeConfigure = true;
|
dontUseCmakeConfigure = true;
|
||||||
|
|
||||||
mesonFlags =
|
mesonFlags = [
|
||||||
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
|
(lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks"))
|
||||||
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
|
(lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
|
||||||
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
|
]
|
||||||
++ lib.optionals havePerl (
|
++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
|
||||||
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
|
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
|
||||||
)
|
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
|
||||||
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
|
++ lib.optionals havePerl (
|
||||||
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
|
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
|
||||||
|
)
|
||||||
|
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
|
||||||
|
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
|
||||||
|
|
||||||
nativeBuildInputs =
|
nativeBuildInputs =
|
||||||
let
|
let
|
||||||
inputs =
|
inputs =
|
||||||
dedupByString (v: "${v}") (
|
dedupByString (v: "${v}") (
|
||||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
|
lib.filter (x: !isInternal x) (
|
||||||
|
lib.lists.concatMap (
|
||||||
|
# Nix manual has a build-time dependency on nix, but we
|
||||||
|
# don't want to do a native build just to enter the ross
|
||||||
|
# dev shell.
|
||||||
|
#
|
||||||
|
# TODO: think of a more principled fix for this.
|
||||||
|
c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs
|
||||||
|
) activeComponents
|
||||||
|
)
|
||||||
)
|
)
|
||||||
++ lib.optional (
|
++ lib.optional (
|
||||||
!buildCanExecuteHost
|
!buildCanExecuteHost
|
||||||
@@ -305,8 +321,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
|||||||
pkgs.buildPackages.nixfmt-rfc-style
|
pkgs.buildPackages.nixfmt-rfc-style
|
||||||
pkgs.buildPackages.shellcheck
|
pkgs.buildPackages.shellcheck
|
||||||
pkgs.buildPackages.include-what-you-use
|
pkgs.buildPackages.include-what-you-use
|
||||||
pkgs.buildPackages.gdb
|
|
||||||
]
|
]
|
||||||
|
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
|
||||||
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
|
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
|
||||||
lib.hiPrio pkgs.buildPackages.clang-tools
|
lib.hiPrio pkgs.buildPackages.clang-tools
|
||||||
)
|
)
|
||||||
@@ -322,13 +338,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
buildInputs = [
|
buildInputs =
|
||||||
pkgs.gbenchmark
|
# TODO change Nixpkgs to mark gbenchmark as building on Windows
|
||||||
]
|
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
|
||||||
++ dedupByString (v: "${v}") (
|
++ dedupByString (v: "${v}") (
|
||||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
||||||
)
|
)
|
||||||
++ lib.optional havePerl pkgs.perl;
|
++ lib.optional havePerl pkgs.perl;
|
||||||
|
|
||||||
propagatedBuildInputs = dedupByString (v: "${v}") (
|
propagatedBuildInputs = dedupByString (v: "${v}") (
|
||||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
|
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
|
||||||
|
|||||||
@@ -62,9 +62,11 @@ schemas = [
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
'stem' : 'build-trace-entry',
|
'stem' : 'build-trace-entry',
|
||||||
'schema' : schema_dir / 'build-trace-entry-v1.yaml',
|
'schema' : schema_dir / 'build-trace-entry-v2.yaml',
|
||||||
'files' : [
|
'files' : [
|
||||||
'simple.json',
|
'simple.json',
|
||||||
|
# The field is no longer supported, but we want to show that we
|
||||||
|
# ignore it during parsing.
|
||||||
'with-dependent-realisations.json',
|
'with-dependent-realisations.json',
|
||||||
'with-signature.json',
|
'with-signature.json',
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -741,6 +741,11 @@ public:
|
|||||||
inDebugger = true;
|
inDebugger = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DebuggerGuard(DebuggerGuard &&) = delete;
|
||||||
|
DebuggerGuard(const DebuggerGuard &) = delete;
|
||||||
|
DebuggerGuard & operator=(DebuggerGuard &&) = delete;
|
||||||
|
DebuggerGuard & operator=(const DebuggerGuard &) = delete;
|
||||||
|
|
||||||
~DebuggerGuard()
|
~DebuggerGuard()
|
||||||
{
|
{
|
||||||
inDebugger = false;
|
inDebugger = false;
|
||||||
|
|||||||
@@ -107,6 +107,8 @@ private:
|
|||||||
Bindings & operator=(const Bindings &) = delete;
|
Bindings & operator=(const Bindings &) = delete;
|
||||||
Bindings & operator=(Bindings &&) = delete;
|
Bindings & operator=(Bindings &&) = delete;
|
||||||
|
|
||||||
|
~Bindings() = default;
|
||||||
|
|
||||||
friend class BindingsBuilder;
|
friend class BindingsBuilder;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -164,8 +164,6 @@ public:
|
|||||||
Value ** elems;
|
Value ** elems;
|
||||||
ListBuilder(EvalMemory & mem, size_t size);
|
ListBuilder(EvalMemory & mem, size_t size);
|
||||||
|
|
||||||
// NOTE: Can be noexcept because we are just copying integral values and
|
|
||||||
// raw pointers.
|
|
||||||
ListBuilder(ListBuilder && x) noexcept
|
ListBuilder(ListBuilder && x) noexcept
|
||||||
: size(x.size)
|
: size(x.size)
|
||||||
, inlineElems{x.inlineElems[0], x.inlineElems[1]}
|
, inlineElems{x.inlineElems[0], x.inlineElems[1]}
|
||||||
@@ -173,6 +171,11 @@ public:
|
|||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ListBuilder(const ListBuilder &) = delete;
|
||||||
|
ListBuilder & operator=(ListBuilder &&) = delete;
|
||||||
|
ListBuilder & operator=(const ListBuilder &) = delete;
|
||||||
|
~ListBuilder() = default;
|
||||||
|
|
||||||
Value *& operator[](size_t n)
|
Value *& operator[](size_t n)
|
||||||
{
|
{
|
||||||
return elems[n];
|
return elems[n];
|
||||||
|
|||||||
+18
-2
@@ -4061,6 +4061,8 @@ static RegisterPrimOp primop_sort({
|
|||||||
|
|
||||||
1. Transitivity
|
1. Transitivity
|
||||||
|
|
||||||
|
If a is less than b and b is less than c, then it follows that a is less than c.
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
comparator a b && comparator b c -> comparator a c
|
comparator a b && comparator b c -> comparator a c
|
||||||
```
|
```
|
||||||
@@ -4073,9 +4075,23 @@ static RegisterPrimOp primop_sort({
|
|||||||
|
|
||||||
1. Transitivity of equivalence
|
1. Transitivity of equivalence
|
||||||
|
|
||||||
|
First, two values a and b are considered equivalent with respect to the comparator if:
|
||||||
|
|
||||||
|
```
|
||||||
|
!comparator a b && !comparator b a
|
||||||
|
```
|
||||||
|
|
||||||
|
In other words, neither is considered "less than" the other.
|
||||||
|
|
||||||
|
Transitivity of equivalence means:
|
||||||
|
|
||||||
|
If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c.
|
||||||
|
|
||||||
```nix
|
```nix
|
||||||
let equiv = a: b: (!comparator a b && !comparator b a); in
|
let
|
||||||
equiv a b && equiv b c -> equiv a c
|
equiv = x: y: (!comparator x y && !comparator y x);
|
||||||
|
in
|
||||||
|
equiv a b && equiv b c -> equiv a c
|
||||||
```
|
```
|
||||||
|
|
||||||
If the *comparator* violates any of these properties, then `builtins.sort`
|
If the *comparator* violates any of these properties, then `builtins.sort`
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ public:
|
|||||||
|
|
||||||
ref<GitRepo> openRepo()
|
ref<GitRepo> openRepo()
|
||||||
{
|
{
|
||||||
return GitRepo::openRepo(tmpDir, true, false);
|
return GitRepo::openRepo(tmpDir, {.create = true});
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string getRepoName() const
|
std::string getRepoName() const
|
||||||
|
|||||||
@@ -203,16 +203,19 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
|
|||||||
|
|
||||||
} // extern "C"
|
} // extern "C"
|
||||||
|
|
||||||
static void initRepoAtomically(std::filesystem::path & path, bool bare)
|
static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options)
|
||||||
{
|
{
|
||||||
if (pathExists(path.string()))
|
if (pathExists(path.string()))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
if (!options.create)
|
||||||
|
throw Error("Git repository %s does not exist.", path);
|
||||||
|
|
||||||
std::filesystem::path tmpDir = createTempDir(path.parent_path());
|
std::filesystem::path tmpDir = createTempDir(path.parent_path());
|
||||||
AutoDelete delTmpDir(tmpDir, true);
|
AutoDelete delTmpDir(tmpDir, true);
|
||||||
Repository tmpRepo;
|
Repository tmpRepo;
|
||||||
|
|
||||||
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), bare))
|
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
|
||||||
throw Error("creating Git repository %s: %s", path, git_error_last()->message);
|
throw Error("creating Git repository %s: %s", path, git_error_last()->message);
|
||||||
try {
|
try {
|
||||||
std::filesystem::rename(tmpDir, path);
|
std::filesystem::rename(tmpDir, path);
|
||||||
@@ -234,7 +237,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
|||||||
/** Location of the repository on disk. */
|
/** Location of the repository on disk. */
|
||||||
std::filesystem::path path;
|
std::filesystem::path path;
|
||||||
|
|
||||||
bool bare;
|
Options options;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* libgit2 repository. Note that new objects are not written to disk,
|
* libgit2 repository. Note that new objects are not written to disk,
|
||||||
@@ -255,18 +258,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
|||||||
*/
|
*/
|
||||||
git_odb_backend * packBackend = nullptr;
|
git_odb_backend * packBackend = nullptr;
|
||||||
|
|
||||||
GitRepoImpl(std::filesystem::path _path, bool create, bool bare, bool packfilesOnly = false)
|
GitRepoImpl(std::filesystem::path _path, Options _options)
|
||||||
: path(std::move(_path))
|
: path(std::move(_path))
|
||||||
, bare(bare)
|
, options(_options)
|
||||||
{
|
{
|
||||||
initLibGit2();
|
initLibGit2();
|
||||||
|
|
||||||
initRepoAtomically(path, bare);
|
initRepoAtomically(path, options);
|
||||||
if (git_repository_open(Setter(repo), path.string().c_str()))
|
if (git_repository_open(Setter(repo), path.string().c_str()))
|
||||||
throw Error("opening Git repository %s: %s", path, git_error_last()->message);
|
throw Error("opening Git repository %s: %s", path, git_error_last()->message);
|
||||||
|
|
||||||
ObjectDb odb;
|
ObjectDb odb;
|
||||||
if (packfilesOnly) {
|
if (options.packfilesOnly) {
|
||||||
/* Create a fresh object database because by default the repo also
|
/* Create a fresh object database because by default the repo also
|
||||||
loose object backends. We are not using any of those for the
|
loose object backends. We are not using any of those for the
|
||||||
tarball cache, but libgit2 still does a bunch of unnecessary
|
tarball cache, but libgit2 still does a bunch of unnecessary
|
||||||
@@ -295,7 +298,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
|||||||
if (git_odb_add_backend(odb.get(), mempackBackend, 999))
|
if (git_odb_add_backend(odb.get(), mempackBackend, 999))
|
||||||
throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
|
throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
|
||||||
|
|
||||||
if (packfilesOnly) {
|
if (options.packfilesOnly) {
|
||||||
if (git_repository_set_odb(repo.get(), odb.get()))
|
if (git_repository_set_odb(repo.get(), odb.get()))
|
||||||
throw Error("setting Git object database: %s", git_error_last()->message);
|
throw Error("setting Git object database: %s", git_error_last()->message);
|
||||||
}
|
}
|
||||||
@@ -366,7 +369,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
|||||||
{
|
{
|
||||||
// TODO: as an optimization, it would be nice to include `this` in the pool.
|
// TODO: as an optimization, it would be nice to include `this` in the pool.
|
||||||
return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
|
return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
|
||||||
return make_ref<GitRepoImpl>(path, false, bare);
|
return make_ref<GitRepoImpl>(path, options);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -712,9 +715,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare, bool packfilesOnly)
|
ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Options options)
|
||||||
{
|
{
|
||||||
return make_ref<GitRepoImpl>(path, create, bare, packfilesOnly);
|
return make_ref<GitRepoImpl>(path, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1427,8 +1430,12 @@ namespace fetchers {
|
|||||||
|
|
||||||
ref<GitRepo> Settings::getTarballCache() const
|
ref<GitRepo> Settings::getTarballCache() const
|
||||||
{
|
{
|
||||||
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
|
/* v1: Had either only loose objects or thin packfiles referring to loose objects
|
||||||
return GitRepo::openRepo(repoDir, /*create=*/true, /*bare=*/true, /*packfilesOnly=*/true);
|
* v2: Must have only packfiles with no loose objects. Should get repacked periodically
|
||||||
|
* for optimal packfiles.
|
||||||
|
*/
|
||||||
|
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2";
|
||||||
|
return GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true});
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace fetchers
|
} // namespace fetchers
|
||||||
@@ -1442,7 +1449,7 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path &
|
|||||||
if (i != cache->end())
|
if (i != cache->end())
|
||||||
return i->second;
|
return i->second;
|
||||||
}
|
}
|
||||||
auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo();
|
auto workdirInfo = GitRepo::openRepo(path, {})->getWorkdirInfo();
|
||||||
_cache.lock()->emplace(path, workdirInfo);
|
_cache.lock()->emplace(path, workdirInfo);
|
||||||
return workdirInfo;
|
return workdirInfo;
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-13
@@ -637,11 +637,6 @@ struct GitInputScheme : InputScheme
|
|||||||
url);
|
url);
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we don't check here for the path existence, then we can give libgit2 any directory
|
|
||||||
// and it will initialize them as git directories.
|
|
||||||
if (!pathExists(path)) {
|
|
||||||
throw Error("The path '%s' does not exist.", path);
|
|
||||||
}
|
|
||||||
repoInfo.location = std::filesystem::absolute(path);
|
repoInfo.location = std::filesystem::absolute(path);
|
||||||
} else {
|
} else {
|
||||||
if (url.scheme == "file")
|
if (url.scheme == "file")
|
||||||
@@ -703,7 +698,7 @@ struct GitInputScheme : InputScheme
|
|||||||
if (auto res = cache->lookup(key))
|
if (auto res = cache->lookup(key))
|
||||||
return getIntAttr(*res, "lastModified");
|
return getIntAttr(*res, "lastModified");
|
||||||
|
|
||||||
auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev);
|
auto lastModified = GitRepo::openRepo(repoDir, {})->getLastModified(rev);
|
||||||
|
|
||||||
cache->upsert(key, {{"lastModified", lastModified}});
|
cache->upsert(key, {{"lastModified", lastModified}});
|
||||||
|
|
||||||
@@ -726,7 +721,7 @@ struct GitInputScheme : InputScheme
|
|||||||
Activity act(
|
Activity act(
|
||||||
*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg()));
|
*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg()));
|
||||||
|
|
||||||
auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
|
auto revCount = GitRepo::openRepo(repoDir, {})->getRevCount(rev);
|
||||||
|
|
||||||
cache->upsert(key, Attrs{{"revCount", revCount}});
|
cache->upsert(key, Attrs{{"revCount", revCount}});
|
||||||
|
|
||||||
@@ -737,7 +732,7 @@ struct GitInputScheme : InputScheme
|
|||||||
{
|
{
|
||||||
auto head = std::visit(
|
auto head = std::visit(
|
||||||
overloaded{
|
overloaded{
|
||||||
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path)->getWorkdirRef(); },
|
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
|
||||||
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
|
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
|
||||||
repoInfo.location);
|
repoInfo.location);
|
||||||
if (!head) {
|
if (!head) {
|
||||||
@@ -795,7 +790,7 @@ struct GitInputScheme : InputScheme
|
|||||||
if (auto repoPath = repoInfo.getPath()) {
|
if (auto repoPath = repoInfo.getPath()) {
|
||||||
repoDir = *repoPath;
|
repoDir = *repoPath;
|
||||||
if (!input.getRev())
|
if (!input.getRev())
|
||||||
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
|
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir, {})->resolveRef(ref).gitRev());
|
||||||
} else {
|
} else {
|
||||||
auto repoUrl = std::get<ParsedURL>(repoInfo.location);
|
auto repoUrl = std::get<ParsedURL>(repoInfo.location);
|
||||||
std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
|
std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
|
||||||
@@ -805,7 +800,7 @@ struct GitInputScheme : InputScheme
|
|||||||
std::filesystem::create_directories(cacheDir.parent_path());
|
std::filesystem::create_directories(cacheDir.parent_path());
|
||||||
PathLocks cacheDirLock({cacheDir.string()});
|
PathLocks cacheDirLock({cacheDir.string()});
|
||||||
|
|
||||||
auto repo = GitRepo::openRepo(cacheDir, true, true);
|
auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true});
|
||||||
|
|
||||||
// We need to set the origin so resolving submodule URLs works
|
// We need to set the origin so resolving submodule URLs works
|
||||||
repo->setRemote("origin", repoUrl.to_string());
|
repo->setRemote("origin", repoUrl.to_string());
|
||||||
@@ -876,7 +871,7 @@ struct GitInputScheme : InputScheme
|
|||||||
// the remainder
|
// the remainder
|
||||||
}
|
}
|
||||||
|
|
||||||
auto repo = GitRepo::openRepo(repoDir);
|
auto repo = GitRepo::openRepo(repoDir, {});
|
||||||
|
|
||||||
auto isShallow = repo->isShallow();
|
auto isShallow = repo->isShallow();
|
||||||
|
|
||||||
@@ -963,7 +958,7 @@ struct GitInputScheme : InputScheme
|
|||||||
for (auto & submodule : repoInfo.workdirInfo.submodules)
|
for (auto & submodule : repoInfo.workdirInfo.submodules)
|
||||||
repoInfo.workdirInfo.files.insert(submodule.path);
|
repoInfo.workdirInfo.files.insert(submodule.path);
|
||||||
|
|
||||||
auto repo = GitRepo::openRepo(repoPath, false, false);
|
auto repo = GitRepo::openRepo(repoPath, {});
|
||||||
|
|
||||||
auto exportIgnore = getExportIgnoreAttr(input);
|
auto exportIgnore = getExportIgnoreAttr(input);
|
||||||
|
|
||||||
@@ -1003,7 +998,7 @@ struct GitInputScheme : InputScheme
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!repoInfo.workdirInfo.isDirty) {
|
if (!repoInfo.workdirInfo.isDirty) {
|
||||||
auto repo = GitRepo::openRepo(repoPath);
|
auto repo = GitRepo::openRepo(repoPath, {});
|
||||||
|
|
||||||
if (auto ref = repo->getWorkdirRef())
|
if (auto ref = repo->getWorkdirRef())
|
||||||
input.attrs.insert_or_assign("ref", *ref);
|
input.attrs.insert_or_assign("ref", *ref);
|
||||||
|
|||||||
@@ -32,8 +32,14 @@ struct GitRepo
|
|||||||
{
|
{
|
||||||
virtual ~GitRepo() {}
|
virtual ~GitRepo() {}
|
||||||
|
|
||||||
static ref<GitRepo>
|
struct Options
|
||||||
openRepo(const std::filesystem::path & path, bool create = false, bool bare = false, bool packfilesOnly = false);
|
{
|
||||||
|
bool create = false;
|
||||||
|
bool bare = false;
|
||||||
|
bool packfilesOnly = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
static ref<GitRepo> openRepo(const std::filesystem::path & path, Options options);
|
||||||
|
|
||||||
virtual uint64_t getRevCount(const Hash & rev) = 0;
|
virtual uint64_t getRevCount(const Hash & rev) = 0;
|
||||||
|
|
||||||
|
|||||||
@@ -88,25 +88,38 @@ public:
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
TEST_F(FIXTURE, NAME##_read) \
|
TEST_F(FIXTURE, NAME##_read) \
|
||||||
{ \
|
{ \
|
||||||
readProtoTest(STEM, VERSION, VALUE); \
|
readProtoTest(STEM, VERSION, VALUE); \
|
||||||
} \
|
|
||||||
TEST_F(FIXTURE, NAME##_write) \
|
|
||||||
{ \
|
|
||||||
writeProtoTest(STEM, VERSION, VALUE); \
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
TEST_F(FIXTURE, NAME##_write) \
|
||||||
TEST_F(FIXTURE, NAME##_json_read) \
|
{ \
|
||||||
{ \
|
writeProtoTest(STEM, VERSION, VALUE); \
|
||||||
readJsonTest(STEM, VALUE); \
|
|
||||||
} \
|
|
||||||
TEST_F(FIXTURE, NAME##_json_write) \
|
|
||||||
{ \
|
|
||||||
writeJsonTest(STEM, VALUE); \
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
|
||||||
|
|
||||||
|
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
TEST_F(FIXTURE, NAME##_json_read) \
|
||||||
|
{ \
|
||||||
|
readJsonTest(STEM, VALUE); \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
TEST_F(FIXTURE, NAME##_json_write) \
|
||||||
|
{ \
|
||||||
|
writeJsonTest(STEM, VALUE); \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||||
|
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -47,24 +47,30 @@ public:
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||||
TEST_F(CommonProtoTest, NAME##_read) \
|
TEST_F(CommonProtoTest, NAME##_read) \
|
||||||
{ \
|
{ \
|
||||||
readProtoTest(STEM, VALUE); \
|
readProtoTest(STEM, VALUE); \
|
||||||
} \
|
} \
|
||||||
TEST_F(CommonProtoTest, NAME##_write) \
|
TEST_F(CommonProtoTest, NAME##_json_read) \
|
||||||
{ \
|
{ \
|
||||||
writeProtoTest(STEM, VALUE); \
|
readJsonTest(STEM, VALUE); \
|
||||||
} \
|
|
||||||
TEST_F(CommonProtoTest, NAME##_json_read) \
|
|
||||||
{ \
|
|
||||||
readJsonTest(STEM, VALUE); \
|
|
||||||
} \
|
|
||||||
TEST_F(CommonProtoTest, NAME##_json_write) \
|
|
||||||
{ \
|
|
||||||
writeJsonTest(STEM, VALUE); \
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||||
|
TEST_F(CommonProtoTest, NAME##_write) \
|
||||||
|
{ \
|
||||||
|
writeProtoTest(STEM, VALUE); \
|
||||||
|
} \
|
||||||
|
TEST_F(CommonProtoTest, NAME##_json_write) \
|
||||||
|
{ \
|
||||||
|
writeJsonTest(STEM, VALUE); \
|
||||||
|
}
|
||||||
|
|
||||||
|
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||||
|
READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||||
|
WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
|
||||||
|
|
||||||
CHARACTERIZATION_TEST(
|
CHARACTERIZATION_TEST(
|
||||||
string,
|
string,
|
||||||
"string",
|
"string",
|
||||||
@@ -141,7 +147,7 @@ CHARACTERIZATION_TEST(
|
|||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
|
|
||||||
CHARACTERIZATION_TEST(
|
READ_CHARACTERIZATION_TEST(
|
||||||
realisation_with_deps,
|
realisation_with_deps,
|
||||||
"realisation-with-deps",
|
"realisation-with-deps",
|
||||||
(std::tuple<Realisation>{
|
(std::tuple<Realisation>{
|
||||||
@@ -149,16 +155,6 @@ CHARACTERIZATION_TEST(
|
|||||||
{
|
{
|
||||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||||
.signatures = {"asdf", "qwer"},
|
.signatures = {"asdf", "qwer"},
|
||||||
.dependentRealisations =
|
|
||||||
{
|
|
||||||
{
|
|
||||||
DrvOutput{
|
|
||||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
|
||||||
.outputName = "quux",
|
|
||||||
},
|
|
||||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {
|
||||||
|
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||||
|
"out": {
|
||||||
|
"dependentRealisations": {},
|
||||||
|
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||||
|
"signatures": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am the output of a CA derivation",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||||
|
"narSize": 152,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {
|
||||||
|
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||||
|
"args": [],
|
||||||
|
"builder": "",
|
||||||
|
"env": {},
|
||||||
|
"inputs": {
|
||||||
|
"drvs": {},
|
||||||
|
"srcs": []
|
||||||
|
},
|
||||||
|
"name": "test-ca-drv",
|
||||||
|
"outputs": {
|
||||||
|
"out": {
|
||||||
|
"hashAlgo": "sha256",
|
||||||
|
"method": "nar"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"system": "",
|
||||||
|
"version": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {},
|
||||||
|
"derivations": {
|
||||||
|
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||||
|
"args": [],
|
||||||
|
"builder": "",
|
||||||
|
"env": {},
|
||||||
|
"inputs": {
|
||||||
|
"drvs": {},
|
||||||
|
"srcs": []
|
||||||
|
},
|
||||||
|
"name": "test-ca-drv",
|
||||||
|
"outputs": {
|
||||||
|
"out": {
|
||||||
|
"hashAlgo": "sha256",
|
||||||
|
"method": "nar"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"system": "",
|
||||||
|
"version": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {
|
||||||
|
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||||
|
"out": {
|
||||||
|
"dependentRealisations": {},
|
||||||
|
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||||
|
"signatures": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am the output of a CA derivation",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||||
|
"narSize": 152,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {
|
||||||
|
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||||
|
"out": {
|
||||||
|
"dependentRealisations": {},
|
||||||
|
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||||
|
"signatures": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||||
|
"narSize": 232,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {
|
||||||
|
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||||
|
"args": [],
|
||||||
|
"builder": "",
|
||||||
|
"env": {},
|
||||||
|
"inputs": {
|
||||||
|
"drvs": {
|
||||||
|
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||||
|
"dynamicOutputs": {},
|
||||||
|
"outputs": [
|
||||||
|
"out"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"srcs": []
|
||||||
|
},
|
||||||
|
"name": "root-drv",
|
||||||
|
"outputs": {
|
||||||
|
"out": {
|
||||||
|
"hashAlgo": "sha256",
|
||||||
|
"method": "nar"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"system": "",
|
||||||
|
"version": 4
|
||||||
|
},
|
||||||
|
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||||
|
"args": [],
|
||||||
|
"builder": "",
|
||||||
|
"env": {},
|
||||||
|
"inputs": {
|
||||||
|
"drvs": {},
|
||||||
|
"srcs": []
|
||||||
|
},
|
||||||
|
"name": "dep-drv",
|
||||||
|
"outputs": {
|
||||||
|
"out": {
|
||||||
|
"hashAlgo": "sha256",
|
||||||
|
"method": "nar"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"system": "",
|
||||||
|
"version": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {},
|
||||||
|
"derivations": {
|
||||||
|
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||||
|
"args": [],
|
||||||
|
"builder": "",
|
||||||
|
"env": {},
|
||||||
|
"inputs": {
|
||||||
|
"drvs": {
|
||||||
|
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||||
|
"dynamicOutputs": {},
|
||||||
|
"outputs": [
|
||||||
|
"out"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"srcs": []
|
||||||
|
},
|
||||||
|
"name": "root-drv",
|
||||||
|
"outputs": {
|
||||||
|
"out": {
|
||||||
|
"hashAlgo": "sha256",
|
||||||
|
"method": "nar"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"system": "",
|
||||||
|
"version": 4
|
||||||
|
},
|
||||||
|
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||||
|
"args": [],
|
||||||
|
"builder": "",
|
||||||
|
"env": {},
|
||||||
|
"inputs": {
|
||||||
|
"drvs": {},
|
||||||
|
"srcs": []
|
||||||
|
},
|
||||||
|
"name": "dep-drv",
|
||||||
|
"outputs": {
|
||||||
|
"out": {
|
||||||
|
"hashAlgo": "sha256",
|
||||||
|
"method": "nar"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"system": "",
|
||||||
|
"version": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {
|
||||||
|
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||||
|
"out": {
|
||||||
|
"dependentRealisations": {},
|
||||||
|
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||||
|
"signatures": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||||
|
"out": {
|
||||||
|
"dependentRealisations": {},
|
||||||
|
"outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
|
||||||
|
"signatures": []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||||
|
"narSize": 232,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am the dependency output",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
|
||||||
|
"narSize": 144,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "Hello, world!",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
|
||||||
|
"narSize": 128,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am a dependency",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||||
|
"narSize": 136,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||||
|
"narSize": 184,
|
||||||
|
"references": [
|
||||||
|
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
|
||||||
|
],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
{
|
||||||
|
"buildTrace": {},
|
||||||
|
"config": {
|
||||||
|
"store": "/nix/store"
|
||||||
|
},
|
||||||
|
"contents": {
|
||||||
|
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I am a dependency",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||||
|
"narSize": 136,
|
||||||
|
"references": [],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
|
||||||
|
"contents": {
|
||||||
|
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
|
||||||
|
"executable": false,
|
||||||
|
"type": "regular"
|
||||||
|
},
|
||||||
|
"info": {
|
||||||
|
"ca": {
|
||||||
|
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||||
|
"method": "nar"
|
||||||
|
},
|
||||||
|
"deriver": null,
|
||||||
|
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||||
|
"narSize": 184,
|
||||||
|
"references": [
|
||||||
|
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
|
||||||
|
],
|
||||||
|
"registrationTime": null,
|
||||||
|
"signatures": [],
|
||||||
|
"storeDir": "/nix/store",
|
||||||
|
"ultimate": false,
|
||||||
|
"version": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"derivations": {}
|
||||||
|
}
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
#include "nix/store/derivations.hh"
|
#include "nix/store/derivations.hh"
|
||||||
#include "nix/store/store-api.hh"
|
#include "nix/store/store-api.hh"
|
||||||
#include "nix/util/experimental-features.hh"
|
#include "nix/util/experimental-features.hh"
|
||||||
#include "nix/util/environment-variables.hh"
|
#include "nix/util/tests/test-data.hh"
|
||||||
#include "nix/store/store-open.hh"
|
#include "nix/store/store-open.hh"
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
@@ -50,11 +50,7 @@ static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::st
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Register benchmarks for actual test derivation files if they exist
|
// Register benchmarks for actual test derivation files if they exist
|
||||||
BENCHMARK_CAPTURE(
|
BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
|
||||||
BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
|
BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());
|
||||||
BENCHMARK_CAPTURE(
|
BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
|
||||||
BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
|
BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());
|
||||||
BENCHMARK_CAPTURE(
|
|
||||||
BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
|
|
||||||
BENCHMARK_CAPTURE(
|
|
||||||
BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
|
|
||||||
@@ -85,6 +85,7 @@ sources = files(
|
|||||||
'store-reference.cc',
|
'store-reference.cc',
|
||||||
'uds-remote-store.cc',
|
'uds-remote-store.cc',
|
||||||
'worker-protocol.cc',
|
'worker-protocol.cc',
|
||||||
|
'worker-substitution.cc',
|
||||||
'write-derivation.cc',
|
'write-derivation.cc',
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -123,6 +124,7 @@ if get_option('benchmarks')
|
|||||||
'bench-main.cc',
|
'bench-main.cc',
|
||||||
'derivation-parser-bench.cc',
|
'derivation-parser-bench.cc',
|
||||||
'ref-scan-bench.cc',
|
'ref-scan-bench.cc',
|
||||||
|
'register-valid-paths-bench.cc',
|
||||||
)
|
)
|
||||||
|
|
||||||
benchmark_exe = executable(
|
benchmark_exe = executable(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
#include "nix/store/tests/nix_api_store.hh"
|
#include "nix/store/tests/nix_api_store.hh"
|
||||||
#include "nix/store/globals.hh"
|
#include "nix/store/globals.hh"
|
||||||
#include "nix/util/tests/string_callback.hh"
|
#include "nix/util/tests/string_callback.hh"
|
||||||
|
#include "nix/util/tests/test-data.hh"
|
||||||
#include "nix/util/url.hh"
|
#include "nix/util/url.hh"
|
||||||
|
|
||||||
#include "store-tests-config.hh"
|
#include "store-tests-config.hh"
|
||||||
@@ -302,7 +303,7 @@ public:
|
|||||||
|
|
||||||
store = open_local_store();
|
store = open_local_store();
|
||||||
|
|
||||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
buffer << t.rdbuf();
|
buffer << t.rdbuf();
|
||||||
@@ -357,7 +358,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
|
|||||||
|
|
||||||
auto * store = open_local_store();
|
auto * store = open_local_store();
|
||||||
|
|
||||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||||
|
|
||||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
@@ -404,7 +405,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
|
|||||||
|
|
||||||
auto * store = open_local_store();
|
auto * store = open_local_store();
|
||||||
|
|
||||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
buffer << t.rdbuf();
|
buffer << t.rdbuf();
|
||||||
@@ -449,7 +450,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
|
|||||||
|
|
||||||
auto * store = open_local_store();
|
auto * store = open_local_store();
|
||||||
|
|
||||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
buffer << t.rdbuf();
|
buffer << t.rdbuf();
|
||||||
@@ -494,7 +495,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
|
|||||||
|
|
||||||
auto * store = open_local_store();
|
auto * store = open_local_store();
|
||||||
|
|
||||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
buffer << t.rdbuf();
|
buffer << t.rdbuf();
|
||||||
@@ -870,7 +871,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagati
|
|||||||
*/
|
*/
|
||||||
static std::string load_json_from_test_data(const char * filename)
|
static std::string load_json_from_test_data(const char * filename)
|
||||||
{
|
{
|
||||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||||
std::ifstream t{unitTestData / filename};
|
std::ifstream t{unitTestData / filename};
|
||||||
std::stringstream buffer;
|
std::stringstream buffer;
|
||||||
buffer << t.rdbuf();
|
buffer << t.rdbuf();
|
||||||
|
|||||||
@@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json)
|
|||||||
writeJsonTest(name, value);
|
writeJsonTest(name, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Realisation simple{
|
||||||
|
{
|
||||||
|
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||||
|
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||||
|
HashAlgorithm::SHA256,
|
||||||
|
HashFormat::Base16),
|
||||||
|
.outputName = "foo",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
INSTANTIATE_TEST_SUITE_P(
|
INSTANTIATE_TEST_SUITE_P(
|
||||||
RealisationJSON,
|
RealisationJSON,
|
||||||
RealisationJsonTest,
|
RealisationJsonTest,
|
||||||
([] {
|
::testing::Values(
|
||||||
Realisation simple{
|
std::pair{
|
||||||
{
|
"simple",
|
||||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
simple,
|
||||||
},
|
},
|
||||||
{
|
std::pair{
|
||||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
"with-signature",
|
||||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
[&] {
|
||||||
HashAlgorithm::SHA256,
|
auto r = simple;
|
||||||
HashFormat::Base16),
|
// FIXME actually sign properly
|
||||||
.outputName = "foo",
|
r.signatures = {"asdfasdfasdf"};
|
||||||
},
|
return r;
|
||||||
};
|
}(),
|
||||||
return ::testing::Values(
|
}));
|
||||||
std::pair{
|
|
||||||
"simple",
|
|
||||||
simple,
|
|
||||||
},
|
|
||||||
std::pair{
|
|
||||||
"with-signature",
|
|
||||||
[&] {
|
|
||||||
auto r = simple;
|
|
||||||
// FIXME actually sign properly
|
|
||||||
r.signatures = {"asdfasdfasdf"};
|
|
||||||
return r;
|
|
||||||
}()},
|
|
||||||
std::pair{
|
|
||||||
"with-dependent-realisations",
|
|
||||||
[&] {
|
|
||||||
auto r = simple;
|
|
||||||
r.dependentRealisations = {{
|
|
||||||
{
|
|
||||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
|
||||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
|
||||||
HashAlgorithm::SHA256,
|
|
||||||
HashFormat::Base16),
|
|
||||||
.outputName = "foo",
|
|
||||||
},
|
|
||||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
|
||||||
}};
|
|
||||||
return r;
|
|
||||||
}(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
()));
|
/**
|
||||||
|
* We no longer have a notion of "dependent realisations", but we still
|
||||||
|
* want to parse old realisation files. So make this just be a read test
|
||||||
|
* (no write direction), accordingly.
|
||||||
|
*/
|
||||||
|
TEST_F(RealisationTest, dependent_realisations_from_json)
|
||||||
|
{
|
||||||
|
readJsonTest("with-dependent-realisations", simple);
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
#include <benchmark/benchmark.h>
|
||||||
|
|
||||||
|
#include "nix/store/derivations.hh"
|
||||||
|
#include "nix/store/local-store.hh"
|
||||||
|
#include "nix/store/store-open.hh"
|
||||||
|
#include "nix/util/file-system.hh"
|
||||||
|
#include "nix/util/hash.hh"
|
||||||
|
#include "nix/util/tests/test-data.hh"
|
||||||
|
|
||||||
|
#ifndef _WIN32
|
||||||
|
|
||||||
|
# include <filesystem>
|
||||||
|
# include <fstream>
|
||||||
|
|
||||||
|
using namespace nix;
|
||||||
|
|
||||||
|
static void BM_RegisterValidPathsDerivations(benchmark::State & state)
|
||||||
|
{
|
||||||
|
const int derivationCount = state.range(0);
|
||||||
|
|
||||||
|
for (auto _ : state) {
|
||||||
|
state.PauseTiming();
|
||||||
|
|
||||||
|
auto tmpRoot = createTempDir();
|
||||||
|
auto realStoreDir = tmpRoot / "nix/store";
|
||||||
|
std::filesystem::create_directories(realStoreDir);
|
||||||
|
|
||||||
|
std::shared_ptr<Store> store = openStore(fmt("local?root=%s", tmpRoot.string()));
|
||||||
|
auto localStore = std::dynamic_pointer_cast<LocalStore>(store);
|
||||||
|
if (!localStore)
|
||||||
|
throw Error("expected local store");
|
||||||
|
|
||||||
|
ValidPathInfos infos;
|
||||||
|
for (int i = 0; i < derivationCount; ++i) {
|
||||||
|
std::string drvName = fmt("register-valid-paths-bench-%d", i);
|
||||||
|
auto drvPath = StorePath::random(drvName + ".drv");
|
||||||
|
|
||||||
|
Derivation drv;
|
||||||
|
drv.name = drvName;
|
||||||
|
drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}});
|
||||||
|
drv.platform = "x86_64-linux";
|
||||||
|
drv.builder = "foo";
|
||||||
|
drv.env["out"] = "";
|
||||||
|
drv.fillInOutputPaths(*localStore);
|
||||||
|
|
||||||
|
auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false);
|
||||||
|
|
||||||
|
/* Create an on-disk store object without registering it
|
||||||
|
in the SQLite DB. LocalFSStore::getFSAccessor(path, false)
|
||||||
|
allows reading store objects based on their filesystem
|
||||||
|
presence alone. */
|
||||||
|
std::ofstream out(realStoreDir / std::string(drvPath.to_string()), std::ios::binary);
|
||||||
|
out.write(drvContents.data(), drvContents.size());
|
||||||
|
if (!out)
|
||||||
|
throw SysError("writing derivation to store");
|
||||||
|
|
||||||
|
ValidPathInfo info{drvPath, UnkeyedValidPathInfo(*localStore, Hash::dummy)};
|
||||||
|
info.narSize = drvContents.size();
|
||||||
|
|
||||||
|
infos.emplace(drvPath, std::move(info));
|
||||||
|
}
|
||||||
|
|
||||||
|
state.ResumeTiming();
|
||||||
|
|
||||||
|
localStore->registerValidPaths(infos);
|
||||||
|
|
||||||
|
state.PauseTiming();
|
||||||
|
localStore.reset();
|
||||||
|
store.reset();
|
||||||
|
std::filesystem::remove_all(tmpRoot);
|
||||||
|
state.ResumeTiming();
|
||||||
|
}
|
||||||
|
|
||||||
|
state.SetItemsProcessed(state.iterations() * derivationCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -118,7 +118,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
|||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
|
|
||||||
VERSIONED_CHARACTERIZATION_TEST(
|
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||||
ServeProtoTest,
|
ServeProtoTest,
|
||||||
realisation_with_deps,
|
realisation_with_deps,
|
||||||
"realisation-with-deps",
|
"realisation-with-deps",
|
||||||
@@ -128,16 +128,6 @@ VERSIONED_CHARACTERIZATION_TEST(
|
|||||||
{
|
{
|
||||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||||
.signatures = {"asdf", "qwer"},
|
.signatures = {"asdf", "qwer"},
|
||||||
.dependentRealisations =
|
|
||||||
{
|
|
||||||
{
|
|
||||||
DrvOutput{
|
|
||||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
|
||||||
.outputName = "quux",
|
|
||||||
},
|
|
||||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
|||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
|
|
||||||
VERSIONED_CHARACTERIZATION_TEST(
|
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||||
WorkerProtoTest,
|
WorkerProtoTest,
|
||||||
realisation_with_deps,
|
realisation_with_deps,
|
||||||
"realisation-with-deps",
|
"realisation-with-deps",
|
||||||
@@ -181,16 +181,6 @@ VERSIONED_CHARACTERIZATION_TEST(
|
|||||||
{
|
{
|
||||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||||
.signatures = {"asdf", "qwer"},
|
.signatures = {"asdf", "qwer"},
|
||||||
.dependentRealisations =
|
|
||||||
{
|
|
||||||
{
|
|
||||||
DrvOutput{
|
|
||||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
|
||||||
.outputName = "quux",
|
|
||||||
},
|
|
||||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||||
|
|||||||
@@ -0,0 +1,450 @@
|
|||||||
|
#include <gtest/gtest.h>
|
||||||
|
#include <nlohmann/json.hpp>
|
||||||
|
|
||||||
|
#include "nix/store/build/worker.hh"
|
||||||
|
#include "nix/store/derivations.hh"
|
||||||
|
#include "nix/store/dummy-store-impl.hh"
|
||||||
|
#include "nix/store/globals.hh"
|
||||||
|
#include "nix/util/memory-source-accessor.hh"
|
||||||
|
|
||||||
|
#include "nix/store/tests/libstore.hh"
|
||||||
|
#include "nix/util/tests/json-characterization.hh"
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest<ref<DummyStore>>
|
||||||
|
{
|
||||||
|
std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution";
|
||||||
|
|
||||||
|
protected:
|
||||||
|
ref<DummyStore> dummyStore;
|
||||||
|
ref<DummyStore> substituter;
|
||||||
|
|
||||||
|
WorkerSubstitutionTest()
|
||||||
|
: LibStoreTest([] {
|
||||||
|
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
|
||||||
|
config->readOnly = false;
|
||||||
|
return config->openDummyStore();
|
||||||
|
}())
|
||||||
|
, dummyStore(store.dynamic_pointer_cast<DummyStore>())
|
||||||
|
, substituter([] {
|
||||||
|
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
|
||||||
|
config->readOnly = false;
|
||||||
|
config->isTrusted = true;
|
||||||
|
return config->openDummyStore();
|
||||||
|
}())
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public:
|
||||||
|
std::filesystem::path goldenMaster(std::string_view testStem) const override
|
||||||
|
{
|
||||||
|
return unitTestData / testStem;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void SetUpTestSuite()
|
||||||
|
{
|
||||||
|
initLibStore(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(WorkerSubstitutionTest, singleStoreObject)
|
||||||
|
{
|
||||||
|
// Add a store path to the substituter
|
||||||
|
auto pathInSubstituter = substituter->addToStore(
|
||||||
|
"hello",
|
||||||
|
SourcePath{
|
||||||
|
[] {
|
||||||
|
auto sc = make_ref<MemorySourceAccessor>();
|
||||||
|
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||||
|
.executable = false,
|
||||||
|
.contents = "Hello, world!",
|
||||||
|
}};
|
||||||
|
return sc;
|
||||||
|
}(),
|
||||||
|
},
|
||||||
|
ContentAddressMethod::Raw::NixArchive,
|
||||||
|
HashAlgorithm::SHA256);
|
||||||
|
|
||||||
|
// Snapshot the substituter (has one store object)
|
||||||
|
checkpointJson("single/substituter", substituter);
|
||||||
|
|
||||||
|
// Snapshot the destination store before (should be empty)
|
||||||
|
checkpointJson("../dummy-store/empty", dummyStore);
|
||||||
|
|
||||||
|
// The path should not exist in the destination store yet
|
||||||
|
ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter));
|
||||||
|
|
||||||
|
// Create a worker with our custom substituter
|
||||||
|
Worker worker{*dummyStore, *dummyStore};
|
||||||
|
|
||||||
|
// Override the substituters to use our dummy store substituter
|
||||||
|
ref<Store> substituerAsStore = substituter;
|
||||||
|
worker.getSubstituters = [substituerAsStore]() -> std::list<ref<Store>> { return {substituerAsStore}; };
|
||||||
|
|
||||||
|
// Create a substitution goal for the path
|
||||||
|
auto goal = worker.makePathSubstitutionGoal(pathInSubstituter);
|
||||||
|
|
||||||
|
// Run the worker with -j0 semantics (no local builds, only substitution)
|
||||||
|
// The worker.run() takes a set of goals
|
||||||
|
Goals goals;
|
||||||
|
goals.insert(upcast_goal(goal));
|
||||||
|
worker.run(goals);
|
||||||
|
|
||||||
|
// Snapshot the destination store after (should match the substituter)
|
||||||
|
checkpointJson("single/substituter", dummyStore);
|
||||||
|
|
||||||
|
// The path should now exist in the destination store
|
||||||
|
ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter));
|
||||||
|
|
||||||
|
// Verify the goal succeeded
|
||||||
|
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject)
|
||||||
|
{
|
||||||
|
// First, add a dependency store path to the substituter
|
||||||
|
auto dependencyPath = substituter->addToStore(
|
||||||
|
"dependency",
|
||||||
|
SourcePath{
|
||||||
|
[] {
|
||||||
|
auto sc = make_ref<MemorySourceAccessor>();
|
||||||
|
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||||
|
.executable = false,
|
||||||
|
.contents = "I am a dependency",
|
||||||
|
}};
|
||||||
|
return sc;
|
||||||
|
}(),
|
||||||
|
},
|
||||||
|
ContentAddressMethod::Raw::NixArchive,
|
||||||
|
HashAlgorithm::SHA256);
|
||||||
|
|
||||||
|
// Now add a store path that references the dependency
|
||||||
|
auto mainPath = substituter->addToStore(
|
||||||
|
"main",
|
||||||
|
SourcePath{
|
||||||
|
[&] {
|
||||||
|
auto sc = make_ref<MemorySourceAccessor>();
|
||||||
|
// Include a reference to the dependency path in the contents
|
||||||
|
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||||
|
.executable = false,
|
||||||
|
.contents = "I depend on " + substituter->printStorePath(dependencyPath),
|
||||||
|
}};
|
||||||
|
return sc;
|
||||||
|
}(),
|
||||||
|
},
|
||||||
|
ContentAddressMethod::Raw::NixArchive,
|
||||||
|
HashAlgorithm::SHA256,
|
||||||
|
StorePathSet{dependencyPath});
|
||||||
|
|
||||||
|
// Snapshot the substituter (has two store objects)
|
||||||
|
checkpointJson("with-dep/substituter", substituter);
|
||||||
|
|
||||||
|
// Snapshot the destination store before (should be empty)
|
||||||
|
checkpointJson("../dummy-store/empty", dummyStore);
|
||||||
|
|
||||||
|
// Neither path should exist in the destination store yet
|
||||||
|
ASSERT_FALSE(dummyStore->isValidPath(dependencyPath));
|
||||||
|
ASSERT_FALSE(dummyStore->isValidPath(mainPath));
|
||||||
|
|
||||||
|
// Create a worker with our custom substituter
|
||||||
|
Worker worker{*dummyStore, *dummyStore};
|
||||||
|
|
||||||
|
// Override the substituters to use our dummy store substituter
|
||||||
|
ref<Store> substituterAsStore = substituter;
|
||||||
|
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||||
|
|
||||||
|
// Create a substitution goal for the main path only
|
||||||
|
// The worker should automatically substitute the dependency as well
|
||||||
|
auto goal = worker.makePathSubstitutionGoal(mainPath);
|
||||||
|
|
||||||
|
// Run the worker
|
||||||
|
Goals goals;
|
||||||
|
goals.insert(upcast_goal(goal));
|
||||||
|
worker.run(goals);
|
||||||
|
|
||||||
|
// Snapshot the destination store after (should match the substituter)
|
||||||
|
checkpointJson("with-dep/substituter", dummyStore);
|
||||||
|
|
||||||
|
// Both paths should now exist in the destination store
|
||||||
|
ASSERT_TRUE(dummyStore->isValidPath(dependencyPath));
|
||||||
|
ASSERT_TRUE(dummyStore->isValidPath(mainPath));
|
||||||
|
|
||||||
|
// Verify the goal succeeded
|
||||||
|
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||||
|
{
|
||||||
|
// Enable CA derivations experimental feature
|
||||||
|
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||||
|
|
||||||
|
// Create a CA floating output derivation
|
||||||
|
Derivation drv;
|
||||||
|
drv.name = "test-ca-drv";
|
||||||
|
drv.outputs = {
|
||||||
|
{
|
||||||
|
"out",
|
||||||
|
DerivationOutput{DerivationOutput::CAFloating{
|
||||||
|
.method = ContentAddressMethod::Raw::NixArchive,
|
||||||
|
.hashAlgo = HashAlgorithm::SHA256,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Write the derivation to the destination store
|
||||||
|
auto drvPath = writeDerivation(*dummyStore, drv);
|
||||||
|
|
||||||
|
// Snapshot the destination store before
|
||||||
|
checkpointJson("ca-drv/store-before", dummyStore);
|
||||||
|
|
||||||
|
// Compute the hash modulo of the derivation
|
||||||
|
// For CA floating derivations, the kind is Deferred since outputs aren't known until build
|
||||||
|
auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
|
||||||
|
ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
|
||||||
|
auto drvHash = hashModulo.hashes.at("out");
|
||||||
|
|
||||||
|
// Create the output store object
|
||||||
|
auto outputPath = substituter->addToStore(
|
||||||
|
"test-ca-drv-out",
|
||||||
|
SourcePath{
|
||||||
|
[] {
|
||||||
|
auto sc = make_ref<MemorySourceAccessor>();
|
||||||
|
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||||
|
.executable = false,
|
||||||
|
.contents = "I am the output of a CA derivation",
|
||||||
|
}};
|
||||||
|
return sc;
|
||||||
|
}(),
|
||||||
|
},
|
||||||
|
ContentAddressMethod::Raw::NixArchive,
|
||||||
|
HashAlgorithm::SHA256);
|
||||||
|
|
||||||
|
// Add the realisation (build trace) to the substituter
|
||||||
|
substituter->buildTrace.insert_or_assign(
|
||||||
|
drvHash,
|
||||||
|
std::map<std::string, UnkeyedRealisation>{
|
||||||
|
{
|
||||||
|
"out",
|
||||||
|
UnkeyedRealisation{
|
||||||
|
.outPath = outputPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Snapshot the substituter
|
||||||
|
checkpointJson("ca-drv/substituter", substituter);
|
||||||
|
|
||||||
|
// The realisation should not exist in the destination store yet
|
||||||
|
DrvOutput drvOutput{drvHash, "out"};
|
||||||
|
ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
|
||||||
|
|
||||||
|
// Create a worker with our custom substituter
|
||||||
|
Worker worker{*dummyStore, *dummyStore};
|
||||||
|
|
||||||
|
// Override the substituters to use our dummy store substituter
|
||||||
|
ref<Store> substituterAsStore = substituter;
|
||||||
|
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||||
|
|
||||||
|
// Create a derivation goal for the CA derivation output
|
||||||
|
// The worker should substitute the output rather than building
|
||||||
|
auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true);
|
||||||
|
|
||||||
|
// Run the worker
|
||||||
|
Goals goals;
|
||||||
|
goals.insert(upcast_goal(goal));
|
||||||
|
worker.run(goals);
|
||||||
|
|
||||||
|
// Snapshot the destination store after
|
||||||
|
checkpointJson("ca-drv/store-after", dummyStore);
|
||||||
|
|
||||||
|
// The output path should now exist in the destination store
|
||||||
|
ASSERT_TRUE(dummyStore->isValidPath(outputPath));
|
||||||
|
|
||||||
|
// The realisation should now exist in the destination store
|
||||||
|
auto realisation = dummyStore->queryRealisation(drvOutput);
|
||||||
|
ASSERT_TRUE(realisation);
|
||||||
|
ASSERT_EQ(realisation->outPath, outputPath);
|
||||||
|
|
||||||
|
// Verify the goal succeeded
|
||||||
|
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||||
|
|
||||||
|
// Disable CA derivations experimental feature
|
||||||
|
experimentalFeatureSettings.set("extra-experimental-features", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test for issue #11928: substituting a CA derivation output should not
|
||||||
|
* require fetching the output of an input derivation when that output
|
||||||
|
* is not referenced.
|
||||||
|
*/
|
||||||
|
TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||||
|
{
|
||||||
|
// Enable CA derivations experimental feature
|
||||||
|
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||||
|
|
||||||
|
// Create the dependency CA floating derivation
|
||||||
|
Derivation depDrv;
|
||||||
|
depDrv.name = "dep-drv";
|
||||||
|
depDrv.outputs = {
|
||||||
|
{
|
||||||
|
"out",
|
||||||
|
DerivationOutput{DerivationOutput::CAFloating{
|
||||||
|
.method = ContentAddressMethod::Raw::NixArchive,
|
||||||
|
.hashAlgo = HashAlgorithm::SHA256,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Write the dependency derivation to the destination store
|
||||||
|
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
|
||||||
|
|
||||||
|
// Compute the hash modulo for the dependency derivation
|
||||||
|
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
|
||||||
|
ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
|
||||||
|
auto depDrvHash = depHashModulo.hashes.at("out");
|
||||||
|
|
||||||
|
// Create the output store object for the dependency in the substituter
|
||||||
|
auto depOutputPath = substituter->addToStore(
|
||||||
|
"dep-drv-out",
|
||||||
|
SourcePath{
|
||||||
|
[] {
|
||||||
|
auto sc = make_ref<MemorySourceAccessor>();
|
||||||
|
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||||
|
.executable = false,
|
||||||
|
.contents = "I am the dependency output",
|
||||||
|
}};
|
||||||
|
return sc;
|
||||||
|
}(),
|
||||||
|
},
|
||||||
|
ContentAddressMethod::Raw::NixArchive,
|
||||||
|
HashAlgorithm::SHA256);
|
||||||
|
|
||||||
|
// Add the realisation for the dependency to the substituter
|
||||||
|
substituter->buildTrace.insert_or_assign(
|
||||||
|
depDrvHash,
|
||||||
|
std::map<std::string, UnkeyedRealisation>{
|
||||||
|
{
|
||||||
|
"out",
|
||||||
|
UnkeyedRealisation{
|
||||||
|
.outPath = depOutputPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create the root CA floating derivation that depends on depDrv
|
||||||
|
Derivation rootDrv;
|
||||||
|
rootDrv.name = "root-drv";
|
||||||
|
rootDrv.outputs = {
|
||||||
|
{
|
||||||
|
"out",
|
||||||
|
DerivationOutput{DerivationOutput::CAFloating{
|
||||||
|
.method = ContentAddressMethod::Raw::NixArchive,
|
||||||
|
.hashAlgo = HashAlgorithm::SHA256,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
// Add the dependency derivation as an input
|
||||||
|
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
|
||||||
|
|
||||||
|
// Write the root derivation to the destination store
|
||||||
|
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
|
||||||
|
|
||||||
|
// Snapshot the destination store before
|
||||||
|
checkpointJson("issue-11928/store-before", dummyStore);
|
||||||
|
|
||||||
|
// Compute the hash modulo for the root derivation
|
||||||
|
auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
|
||||||
|
ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
|
||||||
|
auto rootDrvHash = rootHashModulo.hashes.at("out");
|
||||||
|
|
||||||
|
// Create the output store object for the root derivation
|
||||||
|
// Note: it does NOT reference the dependency's output
|
||||||
|
auto rootOutputPath = substituter->addToStore(
|
||||||
|
"root-drv-out",
|
||||||
|
SourcePath{
|
||||||
|
[] {
|
||||||
|
auto sc = make_ref<MemorySourceAccessor>();
|
||||||
|
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||||
|
.executable = false,
|
||||||
|
.contents =
|
||||||
|
"I am the root output. "
|
||||||
|
"I don't reference anything because the other derivation's output is just needed at build time.",
|
||||||
|
}};
|
||||||
|
return sc;
|
||||||
|
}(),
|
||||||
|
},
|
||||||
|
ContentAddressMethod::Raw::NixArchive,
|
||||||
|
HashAlgorithm::SHA256);
|
||||||
|
|
||||||
|
// The DrvOutputs for both derivations
|
||||||
|
DrvOutput depDrvOutput{depDrvHash, "out"};
|
||||||
|
DrvOutput rootDrvOutput{rootDrvHash, "out"};
|
||||||
|
|
||||||
|
// Add the realisation for the root derivation to the substituter
|
||||||
|
substituter->buildTrace.insert_or_assign(
|
||||||
|
rootDrvHash,
|
||||||
|
std::map<std::string, UnkeyedRealisation>{
|
||||||
|
{
|
||||||
|
"out",
|
||||||
|
UnkeyedRealisation{
|
||||||
|
.outPath = rootOutputPath,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Snapshot the substituter
|
||||||
|
// Note: it has realisations for both drvs, but only the root's output store object
|
||||||
|
checkpointJson("issue-11928/substituter", substituter);
|
||||||
|
|
||||||
|
// The realisations should not exist in the destination store yet
|
||||||
|
ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput));
|
||||||
|
ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput));
|
||||||
|
|
||||||
|
// Create a worker with our custom substituter
|
||||||
|
Worker worker{*dummyStore, *dummyStore};
|
||||||
|
|
||||||
|
// Override the substituters to use our dummy store substituter
|
||||||
|
ref<Store> substituterAsStore = substituter;
|
||||||
|
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||||
|
|
||||||
|
// Create a derivation goal for the root derivation output
|
||||||
|
// The worker should substitute the output rather than building
|
||||||
|
auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false);
|
||||||
|
|
||||||
|
// Run the worker
|
||||||
|
Goals goals;
|
||||||
|
goals.insert(upcast_goal(goal));
|
||||||
|
worker.run(goals);
|
||||||
|
|
||||||
|
// Snapshot the destination store after
|
||||||
|
checkpointJson("issue-11928/store-after", dummyStore);
|
||||||
|
|
||||||
|
// The root output path should now exist in the destination store
|
||||||
|
ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath));
|
||||||
|
|
||||||
|
// The root realisation should now exist in the destination store
|
||||||
|
auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput);
|
||||||
|
ASSERT_TRUE(rootRealisation);
|
||||||
|
ASSERT_EQ(rootRealisation->outPath, rootOutputPath);
|
||||||
|
|
||||||
|
// #11928: The dependency's REALISATION should be fetched, because
|
||||||
|
// it is needed to resolve the underlying derivation. Currently the
|
||||||
|
// realisation is not fetched (bug). Once fixed: Change
|
||||||
|
// depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the
|
||||||
|
// ASSERT_EQ
|
||||||
|
auto depRealisation = dummyStore->queryRealisation(depDrvOutput);
|
||||||
|
ASSERT_FALSE(depRealisation);
|
||||||
|
// ASSERT_EQ(depRealisation->outPath, depOutputPath);
|
||||||
|
|
||||||
|
// The dependency's OUTPUT is correctly not fetched (not referenced by root output)
|
||||||
|
ASSERT_FALSE(dummyStore->isValidPath(depOutputPath));
|
||||||
|
|
||||||
|
// Verify the goal succeeded
|
||||||
|
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||||
|
|
||||||
|
// Disable CA derivations experimental feature
|
||||||
|
experimentalFeatureSettings.set("extra-experimental-features", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
+110
-17
@@ -4,15 +4,15 @@
|
|||||||
|
|
||||||
# include <aws/crt/Types.h>
|
# include <aws/crt/Types.h>
|
||||||
# include "nix/store/s3-url.hh"
|
# include "nix/store/s3-url.hh"
|
||||||
# include "nix/util/finally.hh"
|
|
||||||
# include "nix/util/logging.hh"
|
# include "nix/util/logging.hh"
|
||||||
# include "nix/util/url.hh"
|
|
||||||
# include "nix/util/util.hh"
|
|
||||||
|
|
||||||
# include <aws/crt/Api.h>
|
# include <aws/crt/Api.h>
|
||||||
# include <aws/crt/auth/Credentials.h>
|
# include <aws/crt/auth/Credentials.h>
|
||||||
# include <aws/crt/io/Bootstrap.h>
|
# include <aws/crt/io/Bootstrap.h>
|
||||||
|
|
||||||
|
// C library headers for SSO provider support
|
||||||
|
# include <aws/auth/credentials.h>
|
||||||
|
|
||||||
# include <boost/unordered/concurrent_flat_map.hpp>
|
# include <boost/unordered/concurrent_flat_map.hpp>
|
||||||
|
|
||||||
# include <chrono>
|
# include <chrono>
|
||||||
@@ -30,6 +30,46 @@ AwsAuthError::AwsAuthError(int errorCode)
|
|||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper function to wrap a C credentials provider in the C++ interface.
|
||||||
|
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
|
||||||
|
*/
|
||||||
|
static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createWrappedProvider(
|
||||||
|
aws_credentials_provider * rawProvider, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
|
||||||
|
{
|
||||||
|
if (rawProvider == nullptr) {
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto provider = Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider>(allocator, rawProvider, allocator);
|
||||||
|
return std::static_pointer_cast<Aws::Crt::Auth::ICredentialsProvider>(provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an SSO credentials provider using the C library directly.
|
||||||
|
* The C++ wrapper doesn't expose SSO, so we call the C library and wrap the result.
|
||||||
|
* Returns nullptr if SSO provider creation fails (e.g., profile doesn't have SSO config).
|
||||||
|
*/
|
||||||
|
static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createSSOProvider(
|
||||||
|
const std::string & profileName,
|
||||||
|
Aws::Crt::Io::ClientBootstrap * bootstrap,
|
||||||
|
Aws::Crt::Io::TlsContext * tlsContext,
|
||||||
|
Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
|
||||||
|
{
|
||||||
|
aws_credentials_provider_sso_options options;
|
||||||
|
AWS_ZERO_STRUCT(options);
|
||||||
|
|
||||||
|
options.bootstrap = bootstrap->GetUnderlyingHandle();
|
||||||
|
options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr;
|
||||||
|
if (!profileName.empty()) {
|
||||||
|
options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the SSO provider - will return nullptr if SSO isn't configured for this profile
|
||||||
|
// createWrappedProvider handles nullptr gracefully
|
||||||
|
return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator);
|
||||||
|
}
|
||||||
|
|
||||||
static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
|
static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
|
||||||
{
|
{
|
||||||
if (!provider || !provider->IsValid()) {
|
if (!provider || !provider->IsValid()) {
|
||||||
@@ -91,6 +131,22 @@ public:
|
|||||||
logLevel = Aws::Crt::LogLevel::Warn;
|
logLevel = Aws::Crt::LogLevel::Warn;
|
||||||
}
|
}
|
||||||
apiHandle.InitializeLogging(logLevel, stderr);
|
apiHandle.InitializeLogging(logLevel, stderr);
|
||||||
|
|
||||||
|
// Create a shared TLS context for SSO (required for HTTPS connections)
|
||||||
|
auto allocator = Aws::Crt::ApiAllocator();
|
||||||
|
auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator);
|
||||||
|
tlsContext =
|
||||||
|
std::make_shared<Aws::Crt::Io::TlsContext>(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator);
|
||||||
|
if (!tlsContext || !*tlsContext) {
|
||||||
|
warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable");
|
||||||
|
tlsContext = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get bootstrap (lives as long as apiHandle)
|
||||||
|
bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
||||||
|
if (!bootstrap) {
|
||||||
|
throw AwsAuthError("failed to create AWS client bootstrap");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
AwsCredentials getCredentialsRaw(const std::string & profile);
|
AwsCredentials getCredentialsRaw(const std::string & profile);
|
||||||
@@ -111,6 +167,8 @@ public:
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
Aws::Crt::ApiHandle apiHandle;
|
Aws::Crt::ApiHandle apiHandle;
|
||||||
|
std::shared_ptr<Aws::Crt::Io::TlsContext> tlsContext;
|
||||||
|
Aws::Crt::Io::ClientBootstrap * bootstrap;
|
||||||
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
|
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
|
||||||
credentialProviderCache;
|
credentialProviderCache;
|
||||||
};
|
};
|
||||||
@@ -118,23 +176,58 @@ private:
|
|||||||
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
|
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
|
||||||
AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
|
AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
|
||||||
{
|
{
|
||||||
debug(
|
// profileDisplayName is only used for debug logging - SDK uses its default profile
|
||||||
"[pid=%d] creating new AWS credential provider for profile '%s'",
|
// when ProfileNameOverride is not set
|
||||||
getpid(),
|
const char * profileDisplayName = profile.empty() ? "(default)" : profile.c_str();
|
||||||
profile.empty() ? "(default)" : profile.c_str());
|
|
||||||
|
|
||||||
if (profile.empty()) {
|
debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName);
|
||||||
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
|
|
||||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
// Build a custom credential chain: Environment → SSO → Profile → IMDS
|
||||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
|
// This works for both default and named profiles, ensuring consistent behavior
|
||||||
|
// including SSO support and proper TLS context for STS-based role assumption.
|
||||||
|
Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig;
|
||||||
|
auto allocator = Aws::Crt::ApiAllocator();
|
||||||
|
|
||||||
|
auto addProviderToChain = [&](std::string_view name, auto createProvider) {
|
||||||
|
if (auto provider = createProvider()) {
|
||||||
|
chainConfig.Providers.push_back(provider);
|
||||||
|
debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName);
|
||||||
|
} else {
|
||||||
|
debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// 1. Environment variables (highest priority)
|
||||||
|
addProviderToChain("Environment", [&]() {
|
||||||
|
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderEnvironment(allocator);
|
||||||
|
});
|
||||||
|
|
||||||
|
// 2. SSO provider (try it, will fail gracefully if not configured)
|
||||||
|
if (tlsContext) {
|
||||||
|
addProviderToChain("SSO", [&]() { return createSSOProvider(profile, bootstrap, tlsContext.get(), allocator); });
|
||||||
|
} else {
|
||||||
|
debug("Skipped AWS SSO Credential Provider for profile '%s': TLS context unavailable", profileDisplayName);
|
||||||
}
|
}
|
||||||
|
|
||||||
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
|
// 3. Profile provider (for static credentials and role_arn/source_profile with STS)
|
||||||
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
|
addProviderToChain("Profile", [&]() {
|
||||||
// This is safe because the underlying C library will copy this string
|
Aws::Crt::Auth::CredentialsProviderProfileConfig profileConfig;
|
||||||
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
|
profileConfig.Bootstrap = bootstrap;
|
||||||
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
profileConfig.TlsContext = tlsContext.get();
|
||||||
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
|
if (!profile.empty()) {
|
||||||
|
profileConfig.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
|
||||||
|
}
|
||||||
|
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator);
|
||||||
|
});
|
||||||
|
|
||||||
|
// 4. IMDS provider (for EC2 instances, lowest priority)
|
||||||
|
addProviderToChain("IMDS", [&]() {
|
||||||
|
Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig;
|
||||||
|
imdsConfig.Bootstrap = bootstrap;
|
||||||
|
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator);
|
||||||
|
});
|
||||||
|
|
||||||
|
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator);
|
||||||
}
|
}
|
||||||
|
|
||||||
AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
|
AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)
|
||||||
|
|||||||
@@ -71,14 +71,6 @@ void DerivationBuildingGoal::killChild()
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void DerivationBuildingGoal::timedOut(Error && ex)
|
|
||||||
{
|
|
||||||
killChild();
|
|
||||||
// We're not inside a coroutine, hence we can't use co_return here.
|
|
||||||
// Thus we ignore the return value.
|
|
||||||
[[maybe_unused]] Done _ = doneFailure({BuildResult::Failure::TimedOut, std::move(ex)});
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
|
std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
|
||||||
{
|
{
|
||||||
std::string msg;
|
std::string msg;
|
||||||
@@ -443,7 +435,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
|||||||
if (useHook) {
|
if (useHook) {
|
||||||
buildResult.startTime = time(0); // inexact
|
buildResult.startTime = time(0); // inexact
|
||||||
started();
|
started();
|
||||||
co_await Suspend{};
|
|
||||||
|
while (true) {
|
||||||
|
auto event = co_await WaitForChildEvent{};
|
||||||
|
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||||
|
co_await processChildOutput(output->fd, output->data);
|
||||||
|
} else if (std::get_if<ChildEOF>(&event)) {
|
||||||
|
if (!currentLogLine.empty())
|
||||||
|
flushLine();
|
||||||
|
break;
|
||||||
|
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||||
|
killChild();
|
||||||
|
co_return doneFailure(std::move(*timeout));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#ifndef _WIN32
|
#ifndef _WIN32
|
||||||
assert(hook);
|
assert(hook);
|
||||||
@@ -664,7 +669,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
|||||||
worker.childStarted(shared_from_this(), {builderOut}, true, true);
|
worker.childStarted(shared_from_this(), {builderOut}, true, true);
|
||||||
|
|
||||||
started();
|
started();
|
||||||
co_await Suspend{};
|
|
||||||
|
while (true) {
|
||||||
|
auto event = co_await WaitForChildEvent{};
|
||||||
|
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||||
|
co_await processChildOutput(output->fd, output->data);
|
||||||
|
} else if (std::get_if<ChildEOF>(&event)) {
|
||||||
|
if (!currentLogLine.empty())
|
||||||
|
flushLine();
|
||||||
|
break;
|
||||||
|
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||||
|
killChild();
|
||||||
|
co_return doneFailure(std::move(*timeout));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
trace("build done");
|
trace("build done");
|
||||||
|
|
||||||
@@ -970,7 +988,7 @@ Path DerivationBuildingGoal::openLogFile()
|
|||||||
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
||||||
|
|
||||||
if (settings.compressLog)
|
if (settings.compressLog)
|
||||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
|
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
|
||||||
else
|
else
|
||||||
logSink = logFileSink;
|
logSink = logFileSink;
|
||||||
|
|
||||||
@@ -997,7 +1015,7 @@ bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
Goal::Co DerivationBuildingGoal::processChildOutput(Descriptor fd, std::string_view data)
|
||||||
{
|
{
|
||||||
// local & `ssh://`-builds are dealt with here.
|
// local & `ssh://`-builds are dealt with here.
|
||||||
auto isWrittenToLog = isReadDesc(fd);
|
auto isWrittenToLog = isReadDesc(fd);
|
||||||
@@ -1005,14 +1023,11 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
|
|||||||
logSize += data.size();
|
logSize += data.size();
|
||||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||||
killChild();
|
killChild();
|
||||||
// We're not inside a coroutine, hence we can't use co_return here.
|
co_return doneFailure(BuildError(
|
||||||
// Thus we ignore the return value.
|
|
||||||
[[maybe_unused]] Done _ = doneFailure(BuildError(
|
|
||||||
BuildResult::Failure::LogLimitExceeded,
|
BuildResult::Failure::LogLimitExceeded,
|
||||||
"%s killed after writing more than %d bytes of log output",
|
"%s killed after writing more than %d bytes of log output",
|
||||||
getName(),
|
getName(),
|
||||||
settings.maxLogSize));
|
settings.maxLogSize));
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto c : data)
|
for (auto c : data)
|
||||||
@@ -1065,13 +1080,7 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
|
|||||||
currentHookLine += c;
|
currentHookLine += c;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
}
|
co_return Return{};
|
||||||
|
|
||||||
void DerivationBuildingGoal::handleEOF(Descriptor fd)
|
|
||||||
{
|
|
||||||
if (!currentLogLine.empty())
|
|
||||||
flushLine();
|
|
||||||
worker.wakeUp(shared_from_this());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void DerivationBuildingGoal::flushLine()
|
void DerivationBuildingGoal::flushLine()
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
#include "nix/store/build/derivation-goal.hh"
|
#include "nix/store/build/derivation-goal.hh"
|
||||||
|
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||||
#include "nix/store/build/derivation-building-goal.hh"
|
#include "nix/store/build/derivation-building-goal.hh"
|
||||||
#include "nix/store/build/derivation-resolution-goal.hh"
|
#include "nix/store/build/derivation-resolution-goal.hh"
|
||||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||||
@@ -100,9 +101,24 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
|||||||
through substitutes. If that doesn't work, we'll build
|
through substitutes. If that doesn't work, we'll build
|
||||||
them. */
|
them. */
|
||||||
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
|
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
|
||||||
if (!checkResult)
|
if (!checkResult) {
|
||||||
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
|
DrvOutput id{outputHash, wantedOutput};
|
||||||
else {
|
auto g = worker.makeDrvOutputSubstitutionGoal(id);
|
||||||
|
waitees.insert(g);
|
||||||
|
co_await await(std::move(waitees));
|
||||||
|
|
||||||
|
if (nrFailed == 0) {
|
||||||
|
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath)));
|
||||||
|
co_await await(std::move(waitees));
|
||||||
|
|
||||||
|
trace("output path substituted");
|
||||||
|
|
||||||
|
if (nrFailed == 0)
|
||||||
|
worker.store.registerDrvOutput({*g->outputInfo, id});
|
||||||
|
else
|
||||||
|
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
auto * cap = getDerivationCA(*drv);
|
auto * cap = getDerivationCA(*drv);
|
||||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
|
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
|
||||||
checkResult->first.outPath,
|
checkResult->first.outPath,
|
||||||
@@ -210,11 +226,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
|||||||
.outputName = wantedOutput,
|
.outputName = wantedOutput,
|
||||||
}};
|
}};
|
||||||
newRealisation.signatures.clear();
|
newRealisation.signatures.clear();
|
||||||
if (!drv->type().isFixed()) {
|
|
||||||
auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
|
|
||||||
newRealisation.dependentRealisations =
|
|
||||||
drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
|
|
||||||
}
|
|
||||||
worker.store.signRealisation(newRealisation);
|
worker.store.signRealisation(newRealisation);
|
||||||
worker.store.registerDrvOutput(newRealisation);
|
worker.store.registerDrvOutput(newRealisation);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
#include "nix/store/build/worker.hh"
|
#include "nix/store/build/worker.hh"
|
||||||
#include "nix/store/build/substitution-goal.hh"
|
#include "nix/store/build/substitution-goal.hh"
|
||||||
#include "nix/util/callback.hh"
|
#include "nix/util/callback.hh"
|
||||||
#include "nix/store/store-open.hh"
|
|
||||||
#include "nix/store/globals.hh"
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
@@ -21,11 +19,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
|||||||
trace("init");
|
trace("init");
|
||||||
|
|
||||||
/* If the derivation already exists, we’re done */
|
/* If the derivation already exists, we’re done */
|
||||||
if (worker.store.queryRealisation(id)) {
|
if ((outputInfo = worker.store.queryRealisation(id))) {
|
||||||
co_return amDone(ecSuccess);
|
co_return amDone(ecSuccess);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
auto subs = worker.getSubstituters();
|
||||||
|
|
||||||
bool substituterFailed = false;
|
bool substituterFailed = false;
|
||||||
|
|
||||||
@@ -66,16 +64,19 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
|||||||
true,
|
true,
|
||||||
false);
|
false);
|
||||||
|
|
||||||
co_await Suspend{};
|
while (true) {
|
||||||
|
auto event = co_await WaitForChildEvent{};
|
||||||
|
if (std::get_if<ChildOutput>(&event)) {
|
||||||
|
// Doesn't process child output
|
||||||
|
} else if (std::get_if<ChildEOF>(&event)) {
|
||||||
|
break;
|
||||||
|
} else if (std::get_if<TimedOut>(&event)) {
|
||||||
|
unreachable();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
worker.childTerminated(this);
|
worker.childTerminated(this);
|
||||||
|
|
||||||
/*
|
|
||||||
* The realisation corresponding to the given output id.
|
|
||||||
* Will be filled once we can get it.
|
|
||||||
*/
|
|
||||||
std::shared_ptr<const UnkeyedRealisation> outputInfo;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
outputInfo = promise->get_future().get();
|
outputInfo = promise->get_future().get();
|
||||||
} catch (std::exception & e) {
|
} catch (std::exception & e) {
|
||||||
@@ -86,45 +87,6 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
|||||||
if (!outputInfo)
|
if (!outputInfo)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
bool failed = false;
|
|
||||||
|
|
||||||
Goals waitees;
|
|
||||||
|
|
||||||
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
|
|
||||||
if (depId != id) {
|
|
||||||
if (auto localOutputInfo = worker.store.queryRealisation(depId);
|
|
||||||
localOutputInfo && localOutputInfo->outPath != depPath) {
|
|
||||||
warn(
|
|
||||||
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
|
|
||||||
"Local: %s\n"
|
|
||||||
"Remote: %s",
|
|
||||||
sub->config.getHumanReadableURI(),
|
|
||||||
depId.to_string(),
|
|
||||||
worker.store.printStorePath(localOutputInfo->outPath),
|
|
||||||
worker.store.printStorePath(depPath));
|
|
||||||
failed = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (failed)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
|
|
||||||
|
|
||||||
co_await await(std::move(waitees));
|
|
||||||
|
|
||||||
trace("output path substituted");
|
|
||||||
|
|
||||||
if (nrFailed > 0) {
|
|
||||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
|
||||||
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
|
|
||||||
}
|
|
||||||
|
|
||||||
worker.store.registerDrvOutput({*outputInfo, id});
|
|
||||||
|
|
||||||
trace("finished");
|
trace("finished");
|
||||||
co_return amDone(ecSuccess);
|
co_return amDone(ecSuccess);
|
||||||
}
|
}
|
||||||
@@ -149,9 +111,4 @@ std::string DrvOutputSubstitutionGoal::key()
|
|||||||
return "a$" + std::string(id.to_string());
|
return "a$" + std::string(id.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
|
|
||||||
{
|
|
||||||
worker.wakeUp(shared_from_this());
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -4,8 +4,58 @@
|
|||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
TimedOut::TimedOut(time_t maxDuration)
|
||||||
|
: BuildError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration)
|
||||||
|
, maxDuration(maxDuration)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
using Co = nix::Goal::Co;
|
using Co = nix::Goal::Co;
|
||||||
using promise_type = nix::Goal::promise_type;
|
using promise_type = nix::Goal::promise_type;
|
||||||
|
using ChildEvents = decltype(promise_type::childEvents);
|
||||||
|
|
||||||
|
void ChildEvents::pushChildEvent(ChildOutput event)
|
||||||
|
{
|
||||||
|
if (childTimeout)
|
||||||
|
return; // Already timed out, ignore
|
||||||
|
childOutputs.push(std::move(event));
|
||||||
|
}
|
||||||
|
|
||||||
|
void ChildEvents::pushChildEvent(ChildEOF event)
|
||||||
|
{
|
||||||
|
if (childTimeout)
|
||||||
|
return; // Already timed out, ignore
|
||||||
|
assert(!childEOF);
|
||||||
|
childEOF = std::move(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ChildEvents::pushChildEvent(TimedOut event)
|
||||||
|
{
|
||||||
|
// Timeout is immediate - flush pending events
|
||||||
|
childOutputs = {};
|
||||||
|
childEOF.reset();
|
||||||
|
childTimeout = std::move(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ChildEvents::hasChildEvent() const
|
||||||
|
{
|
||||||
|
return !childOutputs.empty() || childEOF || childTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
Goal::ChildEvent ChildEvents::popChildEvent()
|
||||||
|
{
|
||||||
|
if (!childOutputs.empty()) {
|
||||||
|
auto event = std::move(childOutputs.front());
|
||||||
|
childOutputs.pop();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
if (childEOF)
|
||||||
|
return *std::exchange(childEOF, std::nullopt);
|
||||||
|
if (childTimeout)
|
||||||
|
return *std::exchange(childTimeout, std::nullopt);
|
||||||
|
unreachable();
|
||||||
|
}
|
||||||
|
|
||||||
using handle_type = nix::Goal::handle_type;
|
using handle_type = nix::Goal::handle_type;
|
||||||
using Suspend = nix::Goal::Suspend;
|
using Suspend = nix::Goal::Suspend;
|
||||||
|
|
||||||
@@ -206,6 +256,27 @@ void Goal::work()
|
|||||||
assert(top_co || exitCode != ecBusy);
|
assert(top_co || exitCode != ecBusy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Goal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||||
|
{
|
||||||
|
assert(top_co);
|
||||||
|
top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}});
|
||||||
|
worker.wakeUp(shared_from_this());
|
||||||
|
}
|
||||||
|
|
||||||
|
void Goal::handleEOF(Descriptor fd)
|
||||||
|
{
|
||||||
|
assert(top_co);
|
||||||
|
top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd});
|
||||||
|
worker.wakeUp(shared_from_this());
|
||||||
|
}
|
||||||
|
|
||||||
|
void Goal::timedOut(TimedOut && ex)
|
||||||
|
{
|
||||||
|
assert(top_co);
|
||||||
|
top_co->handle.promise().childEvents.pushChildEvent(std::move(ex));
|
||||||
|
worker.wakeUp(shared_from_this());
|
||||||
|
}
|
||||||
|
|
||||||
Goal::Co Goal::yield()
|
Goal::Co Goal::yield()
|
||||||
{
|
{
|
||||||
worker.wakeUp(shared_from_this());
|
worker.wakeUp(shared_from_this());
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
#include "nix/store/build/worker.hh"
|
#include "nix/store/build/worker.hh"
|
||||||
#include "nix/store/store-open.hh"
|
|
||||||
#include "nix/store/build/substitution-goal.hh"
|
#include "nix/store/build/substitution-goal.hh"
|
||||||
#include "nix/store/nar-info.hh"
|
#include "nix/store/nar-info.hh"
|
||||||
#include "nix/util/finally.hh"
|
#include "nix/util/finally.hh"
|
||||||
@@ -60,7 +59,7 @@ Goal::Co PathSubstitutionGoal::init()
|
|||||||
throw Error(
|
throw Error(
|
||||||
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
|
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
|
||||||
|
|
||||||
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
auto subs = worker.getSubstituters();
|
||||||
|
|
||||||
bool substituterFailed = false;
|
bool substituterFailed = false;
|
||||||
std::optional<Error> lastStoresException = std::nullopt;
|
std::optional<Error> lastStoresException = std::nullopt;
|
||||||
@@ -258,7 +257,16 @@ Goal::Co PathSubstitutionGoal::tryToRun(
|
|||||||
true,
|
true,
|
||||||
false);
|
false);
|
||||||
|
|
||||||
co_await Suspend{};
|
while (true) {
|
||||||
|
auto event = co_await WaitForChildEvent{};
|
||||||
|
if (std::get_if<ChildOutput>(&event)) {
|
||||||
|
// Substitution doesn't process child output
|
||||||
|
} else if (std::get_if<ChildEOF>(&event)) {
|
||||||
|
break;
|
||||||
|
} else if (std::get_if<TimedOut>(&event)) {
|
||||||
|
unreachable(); // Substitution doesn't use timeouts
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
trace("substitute finished");
|
trace("substitute finished");
|
||||||
|
|
||||||
@@ -310,11 +318,6 @@ Goal::Co PathSubstitutionGoal::tryToRun(
|
|||||||
co_return doneSuccess(BuildResult::Success::Substituted);
|
co_return doneSuccess(BuildResult::Success::Substituted);
|
||||||
}
|
}
|
||||||
|
|
||||||
void PathSubstitutionGoal::handleEOF(Descriptor fd)
|
|
||||||
{
|
|
||||||
worker.wakeUp(shared_from_this());
|
|
||||||
}
|
|
||||||
|
|
||||||
void PathSubstitutionGoal::cleanup()
|
void PathSubstitutionGoal::cleanup()
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
#include "nix/store/local-store.hh"
|
#include "nix/store/local-store.hh"
|
||||||
#include "nix/store/machines.hh"
|
#include "nix/store/machines.hh"
|
||||||
|
#include "nix/store/store-open.hh"
|
||||||
#include "nix/store/build/worker.hh"
|
#include "nix/store/build/worker.hh"
|
||||||
#include "nix/store/build/substitution-goal.hh"
|
#include "nix/store/build/substitution-goal.hh"
|
||||||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||||
@@ -21,6 +22,7 @@ Worker::Worker(Store & store, Store & evalStore)
|
|||||||
, actSubstitutions(*logger, actCopyPaths)
|
, actSubstitutions(*logger, actCopyPaths)
|
||||||
, store(store)
|
, store(store)
|
||||||
, evalStore(evalStore)
|
, evalStore(evalStore)
|
||||||
|
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
|
||||||
{
|
{
|
||||||
nrLocalBuilds = 0;
|
nrLocalBuilds = 0;
|
||||||
nrSubstitutions = 0;
|
nrSubstitutions = 0;
|
||||||
@@ -479,14 +481,13 @@ void Worker::waitForInput()
|
|||||||
|
|
||||||
if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts
|
if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts
|
||||||
&& after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
|
&& after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
|
||||||
goal->timedOut(
|
goal->timedOut(TimedOut(settings.maxSilentTime));
|
||||||
Error("%1% timed out after %2% seconds of silence", goal->getName(), settings.maxSilentTime));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (
|
else if (
|
||||||
goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts
|
goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts
|
||||||
&& after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) {
|
&& after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) {
|
||||||
goal->timedOut(Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout));
|
goal->timedOut(TimedOut(settings.buildTimeout));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1489,8 +1489,6 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static unsigned constexpr expectedJsonVersionDerivation = 4;
|
|
||||||
|
|
||||||
void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
|
void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
|
||||||
{
|
{
|
||||||
res = nlohmann::json::object();
|
res = nlohmann::json::object();
|
||||||
|
|||||||
@@ -30,8 +30,6 @@
|
|||||||
#include <thread>
|
#include <thread>
|
||||||
#include <regex>
|
#include <regex>
|
||||||
|
|
||||||
using namespace std::string_literals;
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
const unsigned int RETRY_TIME_MS_DEFAULT = 250;
|
const unsigned int RETRY_TIME_MS_DEFAULT = 250;
|
||||||
@@ -41,9 +39,27 @@ FileTransferSettings fileTransferSettings;
|
|||||||
|
|
||||||
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
|
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>;
|
||||||
|
using curlMulti = std::unique_ptr<::CURLM, decltype([](::CURLM * multi) { ::curl_multi_cleanup(multi); })>;
|
||||||
|
|
||||||
|
struct curlMultiError : Error
|
||||||
|
{
|
||||||
|
::CURLMcode code;
|
||||||
|
|
||||||
|
curlMultiError(::CURLMcode code)
|
||||||
|
: Error{"unexpected curl multi error: %s", ::curl_multi_strerror(code)}
|
||||||
|
{
|
||||||
|
assert(code != CURLM_OK);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
struct curlFileTransfer : public FileTransfer
|
struct curlFileTransfer : public FileTransfer
|
||||||
{
|
{
|
||||||
CURLM * curlm = 0;
|
curlMulti curlm;
|
||||||
|
|
||||||
std::random_device rd;
|
std::random_device rd;
|
||||||
std::mt19937 mt19937;
|
std::mt19937 mt19937;
|
||||||
@@ -59,8 +75,9 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
CURL * req = 0;
|
CURL * req = 0;
|
||||||
// buffer to accompany the `req` above
|
// buffer to accompany the `req` above
|
||||||
char errbuf[CURL_ERROR_SIZE];
|
char errbuf[CURL_ERROR_SIZE];
|
||||||
bool active = false; // whether the handle has been added to the multi object
|
bool active = false; // whether the handle has been added to the multi object
|
||||||
bool paused = false; // whether the request has been paused previously
|
bool paused = false; // whether the request has been paused previously
|
||||||
|
bool enqueued = false; // whether the request has been added the incoming queue
|
||||||
std::string statusMsg;
|
std::string statusMsg;
|
||||||
|
|
||||||
unsigned int attempt = 0;
|
unsigned int attempt = 0;
|
||||||
@@ -69,7 +86,7 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
has been reached. */
|
has been reached. */
|
||||||
std::chrono::steady_clock::time_point embargo;
|
std::chrono::steady_clock::time_point embargo;
|
||||||
|
|
||||||
struct curl_slist * requestHeaders = 0;
|
curlSList requestHeaders;
|
||||||
|
|
||||||
std::string encoding;
|
std::string encoding;
|
||||||
|
|
||||||
@@ -92,6 +109,15 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
return httpStatus;
|
return httpStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void appendHeaders(const std::string & header)
|
||||||
|
{
|
||||||
|
curlSList tmpSList = curlSList(::curl_slist_append(requestHeaders.get(), requireCString(header)));
|
||||||
|
if (!tmpSList)
|
||||||
|
throw std::bad_alloc();
|
||||||
|
requestHeaders.release();
|
||||||
|
requestHeaders = std::move(tmpSList);
|
||||||
|
}
|
||||||
|
|
||||||
TransferItem(
|
TransferItem(
|
||||||
curlFileTransfer & fileTransfer,
|
curlFileTransfer & fileTransfer,
|
||||||
const FileTransferRequest & request,
|
const FileTransferRequest & request,
|
||||||
@@ -131,13 +157,13 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
{
|
{
|
||||||
result.urls.push_back(request.uri.to_string());
|
result.urls.push_back(request.uri.to_string());
|
||||||
|
|
||||||
requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
appendHeaders("Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||||
if (!request.expectedETag.empty())
|
if (!request.expectedETag.empty())
|
||||||
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
|
appendHeaders("If-None-Match: " + request.expectedETag);
|
||||||
if (!request.mimeType.empty())
|
if (!request.mimeType.empty())
|
||||||
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
|
appendHeaders("Content-Type: " + request.mimeType);
|
||||||
for (auto it = request.headers.begin(); it != request.headers.end(); ++it) {
|
for (auto it = request.headers.begin(); it != request.headers.end(); ++it) {
|
||||||
requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
|
appendHeaders(fmt("%s: %s", it->first, it->second));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,13 +171,11 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
{
|
{
|
||||||
if (req) {
|
if (req) {
|
||||||
if (active)
|
if (active)
|
||||||
curl_multi_remove_handle(fileTransfer.curlm, req);
|
curl_multi_remove_handle(fileTransfer.curlm.get(), req);
|
||||||
curl_easy_cleanup(req);
|
curl_easy_cleanup(req);
|
||||||
}
|
}
|
||||||
if (requestHeaders)
|
|
||||||
curl_slist_free_all(requestHeaders);
|
|
||||||
try {
|
try {
|
||||||
if (!done)
|
if (!done && enqueued)
|
||||||
fail(FileTransferError(
|
fail(FileTransferError(
|
||||||
Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri));
|
Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri));
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
@@ -348,7 +372,7 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
|
return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
|
#if !defined(_WIN32)
|
||||||
static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose)
|
static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose)
|
||||||
{
|
{
|
||||||
unix::closeOnExec(curlfd);
|
unix::closeOnExec(curlfd);
|
||||||
@@ -411,15 +435,11 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
("curl/" LIBCURL_VERSION " Nix/" + nixVersion
|
("curl/" LIBCURL_VERSION " Nix/" + nixVersion
|
||||||
+ (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : ""))
|
+ (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : ""))
|
||||||
.c_str());
|
.c_str());
|
||||||
#if LIBCURL_VERSION_NUM >= 0x072b00
|
|
||||||
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
|
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
|
||||||
#endif
|
|
||||||
#if LIBCURL_VERSION_NUM >= 0x072f00
|
|
||||||
if (fileTransferSettings.enableHttp2)
|
if (fileTransferSettings.enableHttp2)
|
||||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
||||||
else
|
else
|
||||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
|
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
|
||||||
#endif
|
|
||||||
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
|
||||||
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
|
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
|
||||||
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
|
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
|
||||||
@@ -429,7 +449,7 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
|
curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
|
||||||
curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
|
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders.get());
|
||||||
|
|
||||||
if (settings.downloadSpeed.get() > 0)
|
if (settings.downloadSpeed.get() > 0)
|
||||||
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
|
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
|
||||||
@@ -459,10 +479,9 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
if (settings.caFile != "")
|
if (settings.caFile != "")
|
||||||
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
|
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
|
||||||
|
|
||||||
#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
|
#if !defined(_WIN32)
|
||||||
curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
|
curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
|
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
|
||||||
|
|
||||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
||||||
@@ -694,13 +713,6 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
|
|
||||||
Sync<State> state_;
|
Sync<State> state_;
|
||||||
|
|
||||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
|
||||||
/* We can't use a std::condition_variable to wake up the curl
|
|
||||||
thread, because it only monitors file descriptors. So use a
|
|
||||||
pipe instead. */
|
|
||||||
Pipe wakeupPipe;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
std::thread workerThread;
|
std::thread workerThread;
|
||||||
|
|
||||||
curlFileTransfer()
|
curlFileTransfer()
|
||||||
@@ -709,43 +721,35 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
static std::once_flag globalInit;
|
static std::once_flag globalInit;
|
||||||
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
|
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
|
||||||
|
|
||||||
curlm = curl_multi_init();
|
curlm = curlMulti(curl_multi_init());
|
||||||
|
|
||||||
#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0
|
curl_multi_setopt(curlm.get(), CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||||
curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
|
||||||
#endif
|
|
||||||
#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
|
|
||||||
curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
|
||||||
wakeupPipe.create();
|
|
||||||
fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
workerThread = std::thread([&]() { workerThreadEntry(); });
|
workerThread = std::thread([&]() { workerThreadEntry(); });
|
||||||
}
|
}
|
||||||
|
|
||||||
~curlFileTransfer()
|
~curlFileTransfer()
|
||||||
{
|
{
|
||||||
stopWorkerThread();
|
try {
|
||||||
|
stopWorkerThread();
|
||||||
|
} catch (...) {
|
||||||
|
ignoreExceptionInDestructor();
|
||||||
|
}
|
||||||
workerThread.join();
|
workerThread.join();
|
||||||
|
|
||||||
if (curlm)
|
|
||||||
curl_multi_cleanup(curlm);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void stopWorkerThread()
|
void stopWorkerThread()
|
||||||
{
|
{
|
||||||
/* Signal the worker thread to exit. */
|
/* Signal the worker thread to exit. */
|
||||||
{
|
state_.lock()->quit();
|
||||||
auto state(state_.lock());
|
wakeupMulti();
|
||||||
state->quit();
|
}
|
||||||
}
|
|
||||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
void wakeupMulti()
|
||||||
writeFull(wakeupPipe.writeSide.get(), " ", false);
|
{
|
||||||
#endif
|
if (auto ec = ::curl_multi_wakeup(curlm.get()))
|
||||||
|
throw curlMultiError(ec);
|
||||||
}
|
}
|
||||||
|
|
||||||
void workerThreadMain()
|
void workerThreadMain()
|
||||||
@@ -775,32 +779,25 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
|
|
||||||
/* Let curl do its thing. */
|
/* Let curl do its thing. */
|
||||||
int running;
|
int running;
|
||||||
CURLMcode mc = curl_multi_perform(curlm, &running);
|
CURLMcode mc = curl_multi_perform(curlm.get(), &running);
|
||||||
if (mc != CURLM_OK)
|
if (mc != CURLM_OK)
|
||||||
throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
|
throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
|
||||||
|
|
||||||
/* Set the promises of any finished requests. */
|
/* Set the promises of any finished requests. */
|
||||||
CURLMsg * msg;
|
CURLMsg * msg;
|
||||||
int left;
|
int left;
|
||||||
while ((msg = curl_multi_info_read(curlm, &left))) {
|
while ((msg = curl_multi_info_read(curlm.get(), &left))) {
|
||||||
if (msg->msg == CURLMSG_DONE) {
|
if (msg->msg == CURLMSG_DONE) {
|
||||||
auto i = items.find(msg->easy_handle);
|
auto i = items.find(msg->easy_handle);
|
||||||
assert(i != items.end());
|
assert(i != items.end());
|
||||||
i->second->finish(msg->data.result);
|
i->second->finish(msg->data.result);
|
||||||
curl_multi_remove_handle(curlm, i->second->req);
|
curl_multi_remove_handle(curlm.get(), i->second->req);
|
||||||
i->second->active = false;
|
i->second->active = false;
|
||||||
items.erase(i);
|
items.erase(i);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Wait for activity, including wakeup events. */
|
/* Wait for activity, including wakeup events. */
|
||||||
int numfds = 0;
|
|
||||||
struct curl_waitfd extraFDs[1];
|
|
||||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
|
||||||
extraFDs[0].fd = wakeupPipe.readSide.get();
|
|
||||||
extraFDs[0].events = CURL_WAIT_POLLIN;
|
|
||||||
extraFDs[0].revents = 0;
|
|
||||||
#endif
|
|
||||||
long maxSleepTimeMs = items.empty() ? 10000 : 100;
|
long maxSleepTimeMs = items.empty() ? 10000 : 100;
|
||||||
auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point()
|
auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point()
|
||||||
? std::max(
|
? std::max(
|
||||||
@@ -809,23 +806,14 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
nextWakeup - std::chrono::steady_clock::now())
|
nextWakeup - std::chrono::steady_clock::now())
|
||||||
.count())
|
.count())
|
||||||
: maxSleepTimeMs;
|
: maxSleepTimeMs;
|
||||||
vomit("download thread waiting for %d ms", sleepTimeMs);
|
|
||||||
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
|
int numfds = 0;
|
||||||
|
mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds);
|
||||||
if (mc != CURLM_OK)
|
if (mc != CURLM_OK)
|
||||||
throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
|
throw curlMultiError(mc);
|
||||||
|
|
||||||
nextWakeup = std::chrono::steady_clock::time_point();
|
nextWakeup = std::chrono::steady_clock::time_point();
|
||||||
|
|
||||||
/* Add new curl requests from the incoming requests queue,
|
|
||||||
except for requests that are embargoed (waiting for a
|
|
||||||
retry timeout to expire). */
|
|
||||||
if (extraFDs[0].revents & CURL_WAIT_POLLIN) {
|
|
||||||
char buf[1024];
|
|
||||||
auto res = read(extraFDs[0].fd, buf, sizeof(buf));
|
|
||||||
if (res == -1 && errno != EINTR)
|
|
||||||
throw SysError("reading curl wakeup socket");
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<std::shared_ptr<TransferItem>> incoming;
|
std::vector<std::shared_ptr<TransferItem>> incoming;
|
||||||
auto now = std::chrono::steady_clock::now();
|
auto now = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
@@ -848,7 +836,7 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
for (auto & item : incoming) {
|
for (auto & item : incoming) {
|
||||||
debug("starting %s of %s", item->request.noun(), item->request.uri);
|
debug("starting %s of %s", item->request.noun(), item->request.uri);
|
||||||
item->init();
|
item->init();
|
||||||
curl_multi_add_handle(curlm, item->req);
|
curl_multi_add_handle(curlm.get(), item->req);
|
||||||
item->active = true;
|
item->active = true;
|
||||||
items[item->req] = item;
|
items[item->req] = item;
|
||||||
}
|
}
|
||||||
@@ -898,11 +886,10 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
if (state->isQuitting())
|
if (state->isQuitting())
|
||||||
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
|
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
|
||||||
state->incoming.push(item);
|
state->incoming.push(item);
|
||||||
|
item->enqueued = true; /* Now any exceptions should be reported via the callback. */
|
||||||
}
|
}
|
||||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
|
||||||
writeFull(wakeupPipe.writeSide.get(), " ");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
wakeupMulti();
|
||||||
return ItemHandle(static_cast<Item &>(*item));
|
return ItemHandle(static_cast<Item &>(*item));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -922,9 +909,7 @@ struct curlFileTransfer : public FileTransfer
|
|||||||
{
|
{
|
||||||
auto state(state_.lock());
|
auto state(state_.lock());
|
||||||
state->unpause.push_back(std::move(item));
|
state->unpause.push_back(std::move(item));
|
||||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
wakeupMulti();
|
||||||
writeFull(wakeupPipe.writeSide.get(), " ");
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void unpauseTransfer(ItemHandle handle) override
|
void unpauseTransfer(ItemHandle handle) override
|
||||||
|
|||||||
@@ -79,7 +79,7 @@ struct DerivationBuilderParams
|
|||||||
*/
|
*/
|
||||||
const StorePathSet & inputPaths;
|
const StorePathSet & inputPaths;
|
||||||
|
|
||||||
const std::map<std::string, InitialOutput> & initialOutputs;
|
const std::map<std::string, InitialOutput> initialOutputs;
|
||||||
|
|
||||||
const BuildMode & buildMode;
|
const BuildMode & buildMode;
|
||||||
|
|
||||||
|
|||||||
@@ -100,8 +100,6 @@ private:
|
|||||||
|
|
||||||
std::map<ActivityId, Activity> builderActivities;
|
std::map<ActivityId, Activity> builderActivities;
|
||||||
|
|
||||||
void timedOut(Error && ex) override;
|
|
||||||
|
|
||||||
std::string key() override;
|
std::string key() override;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -129,10 +127,10 @@ private:
|
|||||||
bool isReadDesc(Descriptor fd);
|
bool isReadDesc(Descriptor fd);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Callback used by the worker to write to the log.
|
* Process output from a child process.
|
||||||
*/
|
*/
|
||||||
void handleChildOutput(Descriptor fd, std::string_view data) override;
|
Co processChildOutput(Descriptor fd, std::string_view data);
|
||||||
void handleEOF(Descriptor fd) override;
|
|
||||||
void flushLine();
|
void flushLine();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -52,11 +52,6 @@ struct DerivationGoal : public Goal
|
|||||||
bool storeDerivation);
|
bool storeDerivation);
|
||||||
~DerivationGoal() = default;
|
~DerivationGoal() = default;
|
||||||
|
|
||||||
void timedOut(Error && ex) override
|
|
||||||
{
|
|
||||||
unreachable();
|
|
||||||
};
|
|
||||||
|
|
||||||
std::string key() override;
|
std::string key() override;
|
||||||
|
|
||||||
JobCategory jobCategory() const override
|
JobCategory jobCategory() const override
|
||||||
|
|||||||
@@ -43,8 +43,6 @@ struct DerivationResolutionGoal : public Goal
|
|||||||
*/
|
*/
|
||||||
std::unique_ptr<std::pair<StorePath, BasicDerivation>> resolvedDrv;
|
std::unique_ptr<std::pair<StorePath, BasicDerivation>> resolvedDrv;
|
||||||
|
|
||||||
void timedOut(Error && ex) override {}
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -109,8 +109,6 @@ struct DerivationTrampolineGoal : public Goal
|
|||||||
|
|
||||||
virtual ~DerivationTrampolineGoal();
|
virtual ~DerivationTrampolineGoal();
|
||||||
|
|
||||||
void timedOut(Error && ex) override {}
|
|
||||||
|
|
||||||
std::string key() override;
|
std::string key() override;
|
||||||
|
|
||||||
JobCategory jobCategory() const override
|
JobCategory jobCategory() const override
|
||||||
|
|||||||
@@ -14,11 +14,14 @@ namespace nix {
|
|||||||
class Worker;
|
class Worker;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Substitution of a derivation output.
|
* Fetch a `Realisation` (drv ⨯ output name -> output path) from a
|
||||||
* This is done in three steps:
|
* substituter.
|
||||||
* 1. Fetch the output info from a substituter
|
*
|
||||||
* 2. Substitute the corresponding output path
|
* If the output store object itself should also be substituted, that is
|
||||||
* 3. Register the output info
|
* the responsibility of the caller to do so.
|
||||||
|
*
|
||||||
|
* @todo rename this `BuidlTraceEntryGoal`, which will make sense
|
||||||
|
* especially once `Realisation` is renamed to `BuildTraceEntry`.
|
||||||
*/
|
*/
|
||||||
class DrvOutputSubstitutionGoal : public Goal
|
class DrvOutputSubstitutionGoal : public Goal
|
||||||
{
|
{
|
||||||
@@ -31,17 +34,16 @@ class DrvOutputSubstitutionGoal : public Goal
|
|||||||
public:
|
public:
|
||||||
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
|
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The realisation corresponding to the given output id.
|
||||||
|
* Will be filled once we can get it.
|
||||||
|
*/
|
||||||
|
std::shared_ptr<const UnkeyedRealisation> outputInfo;
|
||||||
|
|
||||||
Co init();
|
Co init();
|
||||||
|
|
||||||
void timedOut(Error && ex) override
|
|
||||||
{
|
|
||||||
unreachable();
|
|
||||||
};
|
|
||||||
|
|
||||||
std::string key() override;
|
std::string key() override;
|
||||||
|
|
||||||
void handleEOF(Descriptor fd) override;
|
|
||||||
|
|
||||||
JobCategory jobCategory() const override
|
JobCategory jobCategory() const override
|
||||||
{
|
{
|
||||||
return JobCategory::Substitution;
|
return JobCategory::Substitution;
|
||||||
|
|||||||
@@ -5,9 +5,18 @@
|
|||||||
#include "nix/store/build-result.hh"
|
#include "nix/store/build-result.hh"
|
||||||
|
|
||||||
#include <coroutine>
|
#include <coroutine>
|
||||||
|
#include <queue>
|
||||||
|
#include <variant>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
struct TimedOut : BuildError
|
||||||
|
{
|
||||||
|
time_t maxDuration;
|
||||||
|
|
||||||
|
TimedOut(time_t maxDuration);
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Forward definition.
|
* Forward definition.
|
||||||
*/
|
*/
|
||||||
@@ -138,6 +147,29 @@ public:
|
|||||||
friend Goal;
|
friend Goal;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event types for child process communication, delivered via coroutines.
|
||||||
|
*/
|
||||||
|
struct ChildOutput
|
||||||
|
{
|
||||||
|
Descriptor fd;
|
||||||
|
std::string data;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ChildEOF
|
||||||
|
{
|
||||||
|
Descriptor fd;
|
||||||
|
};
|
||||||
|
|
||||||
|
using ChildEvent = std::variant<ChildOutput, ChildEOF, TimedOut>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tag type for `co_await`-ing child events.
|
||||||
|
* Returns a `ChildEvent` when resumed.
|
||||||
|
*/
|
||||||
|
struct WaitForChildEvent
|
||||||
|
{};
|
||||||
|
|
||||||
// forward declaration of promise_type, see below
|
// forward declaration of promise_type, see below
|
||||||
struct promise_type;
|
struct promise_type;
|
||||||
|
|
||||||
@@ -276,6 +308,28 @@ public:
|
|||||||
*/
|
*/
|
||||||
bool alive = true;
|
bool alive = true;
|
||||||
|
|
||||||
|
class
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Structured queue of child events:
|
||||||
|
* - outputs: stream of data from child
|
||||||
|
* - eof: optional end-of-stream marker
|
||||||
|
* - timeout: optional timeout that flushes/overrides other events
|
||||||
|
*/
|
||||||
|
std::queue<ChildOutput> childOutputs;
|
||||||
|
std::optional<ChildEOF> childEOF;
|
||||||
|
std::optional<TimedOut> childTimeout;
|
||||||
|
|
||||||
|
public:
|
||||||
|
|
||||||
|
void pushChildEvent(ChildOutput event);
|
||||||
|
void pushChildEvent(ChildEOF event);
|
||||||
|
void pushChildEvent(TimedOut event);
|
||||||
|
bool hasChildEvent() const;
|
||||||
|
ChildEvent popChildEvent();
|
||||||
|
|
||||||
|
} childEvents;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The awaiter used by @ref final_suspend.
|
* The awaiter used by @ref final_suspend.
|
||||||
*/
|
*/
|
||||||
@@ -369,13 +423,66 @@ public:
|
|||||||
return static_cast<Co &&>(co);
|
return static_cast<Co &&>(co);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Awaiter for @ref Suspend. Always suspends, but asserts
|
||||||
|
* there are no pending child events (those should be
|
||||||
|
* consumed first via @ref WaitForChildEvent).
|
||||||
|
*/
|
||||||
|
struct SuspendAwaiter
|
||||||
|
{
|
||||||
|
promise_type & promise;
|
||||||
|
|
||||||
|
bool await_ready()
|
||||||
|
{
|
||||||
|
assert(!promise.childEvents.hasChildEvent());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void await_suspend(handle_type) {}
|
||||||
|
|
||||||
|
void await_resume() {}
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Allows awaiting a @ref Suspend.
|
* Allows awaiting a @ref Suspend.
|
||||||
* Always suspends.
|
* Always suspends.
|
||||||
*/
|
*/
|
||||||
std::suspend_always await_transform(Suspend)
|
SuspendAwaiter await_transform(Suspend)
|
||||||
{
|
{
|
||||||
return {};
|
return SuspendAwaiter{*this};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Awaiter for child events. Suspends and returns the
|
||||||
|
* pending child event when resumed.
|
||||||
|
*/
|
||||||
|
struct ChildEventAwaiter
|
||||||
|
{
|
||||||
|
handle_type handle;
|
||||||
|
|
||||||
|
bool await_ready()
|
||||||
|
{
|
||||||
|
return handle && handle.promise().childEvents.hasChildEvent();
|
||||||
|
}
|
||||||
|
|
||||||
|
void await_suspend(handle_type h)
|
||||||
|
{
|
||||||
|
handle = h;
|
||||||
|
}
|
||||||
|
|
||||||
|
ChildEvent await_resume()
|
||||||
|
{
|
||||||
|
assert(handle);
|
||||||
|
return handle.promise().childEvents.popChildEvent();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows awaiting child events (output, EOF, timeout).
|
||||||
|
*/
|
||||||
|
ChildEventAwaiter await_transform(WaitForChildEvent)
|
||||||
|
{
|
||||||
|
return ChildEventAwaiter{handle_type::from_promise(*this)};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -432,15 +539,23 @@ public:
|
|||||||
|
|
||||||
void work();
|
void work();
|
||||||
|
|
||||||
virtual void handleChildOutput(Descriptor fd, std::string_view data)
|
/**
|
||||||
{
|
* Called by the worker when data is received from a child process.
|
||||||
unreachable();
|
* Stores the event and resumes the coroutine.
|
||||||
}
|
*/
|
||||||
|
void handleChildOutput(Descriptor fd, std::string_view data);
|
||||||
|
|
||||||
virtual void handleEOF(Descriptor fd)
|
/**
|
||||||
{
|
* Called by the worker when EOF is received from a child process.
|
||||||
unreachable();
|
* Stores the event and resumes the coroutine.
|
||||||
}
|
*/
|
||||||
|
void handleEOF(Descriptor fd);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called by the worker when a build times out.
|
||||||
|
* Stores the event and resumes the coroutine.
|
||||||
|
*/
|
||||||
|
void timedOut(TimedOut && ex);
|
||||||
|
|
||||||
void trace(std::string_view s);
|
void trace(std::string_view s);
|
||||||
|
|
||||||
@@ -449,13 +564,6 @@ public:
|
|||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Callback in case of a timeout. It should wake up its waiters,
|
|
||||||
* get rid of any running child processes that are being monitored
|
|
||||||
* by the worker (important!), etc.
|
|
||||||
*/
|
|
||||||
virtual void timedOut(Error && ex) = 0;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Used for comparisons. The order matters a bit for scheduling. We
|
* Used for comparisons. The order matters a bit for scheduling. We
|
||||||
* want:
|
* want:
|
||||||
|
|||||||
@@ -53,11 +53,6 @@ public:
|
|||||||
std::optional<ContentAddress> ca = std::nullopt);
|
std::optional<ContentAddress> ca = std::nullopt);
|
||||||
~PathSubstitutionGoal();
|
~PathSubstitutionGoal();
|
||||||
|
|
||||||
void timedOut(Error && ex) override
|
|
||||||
{
|
|
||||||
unreachable();
|
|
||||||
};
|
|
||||||
|
|
||||||
std::string key() override
|
std::string key() override
|
||||||
{
|
{
|
||||||
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
|
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
|
||||||
@@ -72,12 +67,6 @@ public:
|
|||||||
StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
|
StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
|
||||||
Co finished();
|
Co finished();
|
||||||
|
|
||||||
/**
|
|
||||||
* Callback used by the worker to write to the log.
|
|
||||||
*/
|
|
||||||
void handleChildOutput(Descriptor fd, std::string_view data) override {};
|
|
||||||
void handleEOF(Descriptor fd) override;
|
|
||||||
|
|
||||||
/* Called by destructor, can't be overridden */
|
/* Called by destructor, can't be overridden */
|
||||||
void cleanup() override final;
|
void cleanup() override final;
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
#include "nix/store/realisation.hh"
|
#include "nix/store/realisation.hh"
|
||||||
#include "nix/util/muxable-pipe.hh"
|
#include "nix/util/muxable-pipe.hh"
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
#include <future>
|
#include <future>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
@@ -171,6 +172,14 @@ public:
|
|||||||
Store & store;
|
Store & store;
|
||||||
Store & evalStore;
|
Store & evalStore;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Function to get the substituters to use for path substitution.
|
||||||
|
*
|
||||||
|
* Defaults to `getDefaultSubstituters`. This allows tests to
|
||||||
|
* inject custom substituters.
|
||||||
|
*/
|
||||||
|
std::function<std::list<ref<Store>>()> getSubstituters;
|
||||||
|
|
||||||
#ifndef _WIN32 // TODO Enable building on Windows
|
#ifndef _WIN32 // TODO Enable building on Windows
|
||||||
std::unique_ptr<HookInstance> hook;
|
std::unique_ptr<HookInstance> hook;
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -586,6 +586,12 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
|
|||||||
*/
|
*/
|
||||||
std::string hashPlaceholder(const OutputNameView outputName);
|
std::string hashPlaceholder(const OutputNameView outputName);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The expected JSON version for derivation serialization.
|
||||||
|
* Used by `nix derivation show` and `nix derivation add`.
|
||||||
|
*/
|
||||||
|
constexpr unsigned expectedJsonVersionDerivation = 4;
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
|
|
||||||
JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)
|
JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)
|
||||||
|
|||||||
@@ -1143,7 +1143,7 @@ public:
|
|||||||
|
|
||||||
Setting<std::string> netrcFile{
|
Setting<std::string> netrcFile{
|
||||||
this,
|
this,
|
||||||
fmt("%s/%s", nixConfDir, "netrc"),
|
(nixConfDir / "netrc").string(),
|
||||||
"netrc-file",
|
"netrc-file",
|
||||||
R"(
|
R"(
|
||||||
If set to an absolute path to a `netrc` file, Nix uses the HTTP
|
If set to an absolute path to a `netrc` file, Nix uses the HTTP
|
||||||
|
|||||||
@@ -420,7 +420,7 @@ private:
|
|||||||
|
|
||||||
uint64_t queryValidPathId(State & state, const StorePath & path);
|
uint64_t queryValidPathId(State & state, const StorePath & path);
|
||||||
|
|
||||||
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
|
uint64_t addValidPath(State & state, const ValidPathInfo & info);
|
||||||
|
|
||||||
void invalidatePath(State & state, const StorePath & path);
|
void invalidatePath(State & state, const StorePath & path);
|
||||||
|
|
||||||
|
|||||||
@@ -33,6 +33,22 @@ private:
|
|||||||
public:
|
public:
|
||||||
PathLocks();
|
PathLocks();
|
||||||
PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
|
PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
|
||||||
|
|
||||||
|
PathLocks(PathLocks && other) noexcept
|
||||||
|
: fds(std::exchange(other.fds, {}))
|
||||||
|
, deletePaths(other.deletePaths)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
PathLocks & operator=(PathLocks && other) noexcept
|
||||||
|
{
|
||||||
|
fds = std::exchange(other.fds, {});
|
||||||
|
deletePaths = other.deletePaths;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
PathLocks(const PathLocks &) = delete;
|
||||||
|
PathLocks & operator=(const PathLocks &) = delete;
|
||||||
bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
|
bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
|
||||||
~PathLocks();
|
~PathLocks();
|
||||||
void unlock();
|
void unlock();
|
||||||
@@ -45,6 +61,10 @@ struct FdLock
|
|||||||
bool acquired = false;
|
bool acquired = false;
|
||||||
|
|
||||||
FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
|
FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
|
||||||
|
FdLock(const FdLock &) = delete;
|
||||||
|
FdLock & operator=(const FdLock &) = delete;
|
||||||
|
FdLock(FdLock &&) = delete;
|
||||||
|
FdLock & operator=(FdLock &&) = delete;
|
||||||
|
|
||||||
~FdLock()
|
~FdLock()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -56,14 +56,6 @@ struct UnkeyedRealisation
|
|||||||
|
|
||||||
StringSet signatures;
|
StringSet signatures;
|
||||||
|
|
||||||
/**
|
|
||||||
* The realisations that are required for the current one to be valid.
|
|
||||||
*
|
|
||||||
* When importing this realisation, the store will first check that all its
|
|
||||||
* dependencies exist, and map to the correct output path
|
|
||||||
*/
|
|
||||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
|
||||||
|
|
||||||
std::string fingerprint(const DrvOutput & key) const;
|
std::string fingerprint(const DrvOutput & key) const;
|
||||||
|
|
||||||
void sign(const DrvOutput & key, const Signer &);
|
void sign(const DrvOutput & key, const Signer &);
|
||||||
@@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation
|
|||||||
|
|
||||||
bool isCompatibleWith(const UnkeyedRealisation & other) const;
|
bool isCompatibleWith(const UnkeyedRealisation & other) const;
|
||||||
|
|
||||||
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
|
|
||||||
|
|
||||||
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
|
|
||||||
|
|
||||||
bool operator==(const Realisation &) const = default;
|
bool operator==(const Realisation &) const = default;
|
||||||
auto operator<=>(const Realisation &) const = default;
|
auto operator<=>(const Realisation &) const = default;
|
||||||
};
|
};
|
||||||
@@ -154,10 +142,6 @@ struct RealisedPath
|
|||||||
*/
|
*/
|
||||||
const StorePath & path() const &;
|
const StorePath & path() const &;
|
||||||
|
|
||||||
void closure(Store & store, Set & ret) const;
|
|
||||||
static void closure(Store & store, const Set & startPaths, Set & ret);
|
|
||||||
Set closure(Store & store) const;
|
|
||||||
|
|
||||||
bool operator==(const RealisedPath &) const = default;
|
bool operator==(const RealisedPath &) const = default;
|
||||||
auto operator<=>(const RealisedPath &) const = default;
|
auto operator<=>(const RealisedPath &) const = default;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1007,9 +1007,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
|
|||||||
|
|
||||||
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
||||||
|
|
||||||
std::map<DrvOutput, StorePath>
|
|
||||||
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
|
|
||||||
|
|
||||||
template<>
|
template<>
|
||||||
struct json_avoids_null<TrustedFlag> : std::true_type
|
struct json_avoids_null<TrustedFlag> : std::true_type
|
||||||
{};
|
{};
|
||||||
|
|||||||
@@ -110,8 +110,6 @@ struct LocalStore::State::Stmts
|
|||||||
SQLiteStmt QueryAllRealisedOutputs;
|
SQLiteStmt QueryAllRealisedOutputs;
|
||||||
SQLiteStmt QueryPathFromHashPart;
|
SQLiteStmt QueryPathFromHashPart;
|
||||||
SQLiteStmt QueryValidPaths;
|
SQLiteStmt QueryValidPaths;
|
||||||
SQLiteStmt QueryRealisationReferences;
|
|
||||||
SQLiteStmt AddRealisationReference;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
LocalStore::LocalStore(ref<const Config> config)
|
LocalStore::LocalStore(ref<const Config> config)
|
||||||
@@ -390,21 +388,6 @@ LocalStore::LocalStore(ref<const Config> config)
|
|||||||
where drvPath = ?
|
where drvPath = ?
|
||||||
;
|
;
|
||||||
)");
|
)");
|
||||||
state->stmts->QueryRealisationReferences.create(
|
|
||||||
state->db,
|
|
||||||
R"(
|
|
||||||
select drvPath, outputName from Realisations
|
|
||||||
join RealisationsRefs on realisationReference = Realisations.id
|
|
||||||
where referrer = ?;
|
|
||||||
)");
|
|
||||||
state->stmts->AddRealisationReference.create(
|
|
||||||
state->db,
|
|
||||||
R"(
|
|
||||||
insert or replace into RealisationsRefs (referrer, realisationReference)
|
|
||||||
values (
|
|
||||||
(select id from Realisations where drvPath = ? and outputName = ?),
|
|
||||||
(select id from Realisations where drvPath = ? and outputName = ?));
|
|
||||||
)");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -654,25 +637,6 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
|||||||
concatStringsSep(" ", info.signatures))
|
concatStringsSep(" ", info.signatures))
|
||||||
.exec();
|
.exec();
|
||||||
}
|
}
|
||||||
for (auto & [outputId, depPath] : info.dependentRealisations) {
|
|
||||||
auto localRealisation = queryRealisationCore_(*state, outputId);
|
|
||||||
if (!localRealisation)
|
|
||||||
throw Error(
|
|
||||||
"unable to register the derivation '%s' as it "
|
|
||||||
"depends on the non existent '%s'",
|
|
||||||
info.id.to_string(),
|
|
||||||
outputId.to_string());
|
|
||||||
if (localRealisation->second.outPath != depPath)
|
|
||||||
throw Error(
|
|
||||||
"unable to register the derivation '%s' as it "
|
|
||||||
"depends on a realisation of '%s' that doesn’t"
|
|
||||||
"match what we have locally",
|
|
||||||
info.id.to_string(),
|
|
||||||
outputId.to_string());
|
|
||||||
state->stmts->AddRealisationReference
|
|
||||||
.use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
|
|
||||||
.exec();
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -683,7 +647,7 @@ void LocalStore::cacheDrvOutputMapping(
|
|||||||
[&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); });
|
[&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); });
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs)
|
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info)
|
||||||
{
|
{
|
||||||
if (info.ca.has_value() && !info.isContentAddressed(*this))
|
if (info.ca.has_value() && !info.isContentAddressed(*this))
|
||||||
throw Error(
|
throw Error(
|
||||||
@@ -704,17 +668,16 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo
|
|||||||
efficiently query whether a path is an output of some
|
efficiently query whether a path is an output of some
|
||||||
derivation. */
|
derivation. */
|
||||||
if (info.path.isDerivation()) {
|
if (info.path.isDerivation()) {
|
||||||
auto drv = readInvalidDerivation(info.path);
|
auto parsedDrv = readInvalidDerivation(info.path);
|
||||||
|
|
||||||
/* Verify that the output paths in the derivation are correct
|
/* Verify that the output paths in the derivation are correct
|
||||||
(i.e., follow the scheme for computing output paths from
|
(i.e., follow the scheme for computing output paths from
|
||||||
derivations). Note that if this throws an error, then the
|
derivations). Note that if this throws an error, then the
|
||||||
DB transaction is rolled back, so the path validity
|
DB transaction is rolled back, so the path validity
|
||||||
registration above is undone. */
|
registration above is undone. */
|
||||||
if (checkOutputs)
|
parsedDrv.checkInvariants(*this, info.path);
|
||||||
drv.checkInvariants(*this, info.path);
|
|
||||||
|
|
||||||
for (auto & i : drv.outputsAndOptPaths(*this)) {
|
for (auto & i : parsedDrv.outputsAndOptPaths(*this)) {
|
||||||
/* Floating CA derivations have indeterminate output paths until
|
/* Floating CA derivations have indeterminate output paths until
|
||||||
they are built, so don't register anything in that case */
|
they are built, so don't register anything in that case */
|
||||||
if (i.second.second)
|
if (i.second.second)
|
||||||
@@ -965,7 +928,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||||||
if (isValidPath_(*state, i.path))
|
if (isValidPath_(*state, i.path))
|
||||||
updatePathInfo(*state, i);
|
updatePathInfo(*state, i);
|
||||||
else
|
else
|
||||||
addValidPath(*state, i, false);
|
addValidPath(*state, i);
|
||||||
paths.insert(i.path);
|
paths.insert(i.path);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -975,15 +938,6 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||||||
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Check that the derivation outputs are correct. We can't do
|
|
||||||
this in addValidPath() above, because the references might
|
|
||||||
not be valid yet. */
|
|
||||||
for (auto & [_, i] : infos)
|
|
||||||
if (i.path.isDerivation()) {
|
|
||||||
// FIXME: inefficient; we already loaded the derivation in addValidPath().
|
|
||||||
readInvalidDerivation(i.path).checkInvariants(*this, i.path);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Do a topological sort of the paths. This will throw an
|
/* Do a topological sort of the paths. This will throw an
|
||||||
error if a cycle is detected and roll back the
|
error if a cycle is detected and roll back the
|
||||||
transaction. Cycles can only occur when a derivation
|
transaction. Cycles can only occur when a derivation
|
||||||
@@ -1606,21 +1560,6 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
|
|||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
auto [realisationDbId, res] = *maybeCore;
|
auto [realisationDbId, res] = *maybeCore;
|
||||||
|
|
||||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
|
||||||
auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
|
|
||||||
while (useRealisationRefs.next()) {
|
|
||||||
auto depId = DrvOutput{
|
|
||||||
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
|
|
||||||
useRealisationRefs.getStr(1),
|
|
||||||
};
|
|
||||||
auto dependentRealisation = queryRealisationCore_(state, depId);
|
|
||||||
assert(dependentRealisation); // Enforced by the db schema
|
|
||||||
auto outputPath = dependentRealisation->second.outPath;
|
|
||||||
dependentRealisations.insert({depId, outputPath});
|
|
||||||
}
|
|
||||||
|
|
||||||
res.dependentRealisations = dependentRealisations;
|
|
||||||
|
|
||||||
return {res};
|
return {res};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+28
-22
@@ -39,28 +39,32 @@ deps_public_maybe_subproject = [
|
|||||||
]
|
]
|
||||||
subdir('nix-meson-build-support/subprojects')
|
subdir('nix-meson-build-support/subprojects')
|
||||||
|
|
||||||
run_command(
|
can_link_symlink = false
|
||||||
'ln',
|
native_ln = find_program('ln', required : false, native : true)
|
||||||
'-s',
|
if native_ln.found()
|
||||||
meson.project_build_root() / '__nothing_link_target',
|
run_command(
|
||||||
meson.project_build_root() / '__nothing_symlink',
|
native_ln,
|
||||||
# native doesn't allow dangling symlinks, which the tests require
|
'-s',
|
||||||
env : {'MSYS' : 'winsymlinks:lnk'},
|
meson.project_build_root() / '__nothing_link_target',
|
||||||
check : true,
|
meson.project_build_root() / '__nothing_symlink',
|
||||||
)
|
# native doesn't allow dangling symlinks, which the tests require
|
||||||
can_link_symlink = run_command(
|
env : {'MSYS' : 'winsymlinks:lnk'},
|
||||||
'ln',
|
check : true,
|
||||||
meson.project_build_root() / '__nothing_symlink',
|
)
|
||||||
meson.project_build_root() / '__nothing_hardlink',
|
can_link_symlink = run_command(
|
||||||
check : false,
|
native_ln,
|
||||||
).returncode() == 0
|
meson.project_build_root() / '__nothing_symlink',
|
||||||
run_command(
|
meson.project_build_root() / '__nothing_hardlink',
|
||||||
'rm',
|
check : false,
|
||||||
'-f',
|
).returncode() == 0
|
||||||
meson.project_build_root() / '__nothing_symlink',
|
run_command(
|
||||||
meson.project_build_root() / '__nothing_hardlink',
|
'rm',
|
||||||
check : true,
|
'-f',
|
||||||
)
|
meson.project_build_root() / '__nothing_symlink',
|
||||||
|
meson.project_build_root() / '__nothing_hardlink',
|
||||||
|
check : true,
|
||||||
|
)
|
||||||
|
endif
|
||||||
summary('can hardlink to symlink', can_link_symlink, bool_yn : true)
|
summary('can hardlink to symlink', can_link_symlink, bool_yn : true)
|
||||||
configdata_priv.set('CAN_LINK_SYMLINK', can_link_symlink.to_int())
|
configdata_priv.set('CAN_LINK_SYMLINK', can_link_symlink.to_int())
|
||||||
|
|
||||||
@@ -160,6 +164,8 @@ if s3_aws_auth.enabled()
|
|||||||
deps_other += aws_crt_cpp
|
deps_other += aws_crt_cpp
|
||||||
aws_c_common = cxx.find_library('aws-c-common', required : true)
|
aws_c_common = cxx.find_library('aws-c-common', required : true)
|
||||||
deps_other += aws_c_common
|
deps_other += aws_c_common
|
||||||
|
aws_c_auth = cxx.find_library('aws-c-auth', required : true)
|
||||||
|
deps_other += aws_c_auth
|
||||||
endif
|
endif
|
||||||
|
|
||||||
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())
|
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())
|
||||||
|
|||||||
+9
-82
@@ -23,9 +23,9 @@ void Store::computeFSClosure(
|
|||||||
bool includeOutputs,
|
bool includeOutputs,
|
||||||
bool includeDerivers)
|
bool includeDerivers)
|
||||||
{
|
{
|
||||||
std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
|
std::function<asio::awaitable<StorePathSet>(const StorePath & path)> queryDeps;
|
||||||
if (flipDirection)
|
if (flipDirection)
|
||||||
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
|
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
|
||||||
StorePathSet res;
|
StorePathSet res;
|
||||||
StorePathSet referrers;
|
StorePathSet referrers;
|
||||||
queryReferrers(path, referrers);
|
queryReferrers(path, referrers);
|
||||||
@@ -41,12 +41,14 @@ void Store::computeFSClosure(
|
|||||||
for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
|
for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
|
||||||
if (maybeOutPath && isValidPath(*maybeOutPath))
|
if (maybeOutPath && isValidPath(*maybeOutPath))
|
||||||
res.insert(*maybeOutPath);
|
res.insert(*maybeOutPath);
|
||||||
return res;
|
co_return res;
|
||||||
};
|
};
|
||||||
else
|
else
|
||||||
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
|
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
|
||||||
StorePathSet res;
|
StorePathSet res;
|
||||||
auto info = fut.get();
|
auto info = co_await callbackToAwaitable<ref<const ValidPathInfo>>(
|
||||||
|
[this, path](Callback<ref<const ValidPathInfo>> cb) { queryPathInfo(path, std::move(cb)); });
|
||||||
|
|
||||||
for (auto & ref : info->references)
|
for (auto & ref : info->references)
|
||||||
if (ref != path)
|
if (ref != path)
|
||||||
res.insert(ref);
|
res.insert(ref);
|
||||||
@@ -58,25 +60,9 @@ void Store::computeFSClosure(
|
|||||||
|
|
||||||
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
|
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
|
||||||
res.insert(*info->deriver);
|
res.insert(*info->deriver);
|
||||||
return res;
|
co_return res;
|
||||||
};
|
};
|
||||||
|
computeClosure<StorePath>(startPaths, paths_, queryDeps);
|
||||||
computeClosure<StorePath>(
|
|
||||||
startPaths,
|
|
||||||
paths_,
|
|
||||||
[&](const StorePath & path, std::function<void(std::promise<std::set<StorePath>> &)> processEdges) {
|
|
||||||
std::promise<std::set<StorePath>> promise;
|
|
||||||
std::function<void(std::future<ref<const ValidPathInfo>>)> getDependencies =
|
|
||||||
[&](std::future<ref<const ValidPathInfo>> fut) {
|
|
||||||
try {
|
|
||||||
promise.set_value(queryDeps(path, fut));
|
|
||||||
} catch (...) {
|
|
||||||
promise.set_exception(std::current_exception());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
queryPathInfo(path, getDependencies);
|
|
||||||
processEdges(promise);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void Store::computeFSClosure(
|
void Store::computeFSClosure(
|
||||||
@@ -334,65 +320,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
|
|||||||
result);
|
result);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::map<DrvOutput, StorePath>
|
|
||||||
drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
|
|
||||||
{
|
|
||||||
std::map<DrvOutput, StorePath> res;
|
|
||||||
|
|
||||||
for (const auto & input : inputRealisations) {
|
|
||||||
if (pathReferences.count(input.outPath)) {
|
|
||||||
res.insert({input.id, input.outPath});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::map<DrvOutput, StorePath>
|
|
||||||
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
|
|
||||||
{
|
|
||||||
auto & evalStore = evalStore_ ? *evalStore_ : store;
|
|
||||||
|
|
||||||
std::set<Realisation> inputRealisations;
|
|
||||||
|
|
||||||
auto accumRealisations = [&](this auto & self,
|
|
||||||
const StorePath & inputDrv,
|
|
||||||
const DerivedPathMap<StringSet>::ChildNode & inputNode) -> void {
|
|
||||||
if (!inputNode.value.empty()) {
|
|
||||||
auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
|
|
||||||
for (const auto & outputName : inputNode.value) {
|
|
||||||
auto outputHash = get(outputHashes, outputName);
|
|
||||||
if (!outputHash)
|
|
||||||
throw Error(
|
|
||||||
"output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
|
|
||||||
DrvOutput key{*outputHash, outputName};
|
|
||||||
auto thisRealisation = store.queryRealisation(key);
|
|
||||||
if (!thisRealisation)
|
|
||||||
throw Error(
|
|
||||||
"output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
|
|
||||||
inputRealisations.insert({*thisRealisation, std::move(key)});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!inputNode.value.empty()) {
|
|
||||||
auto d = makeConstantStorePathRef(inputDrv);
|
|
||||||
for (const auto & [outputName, childNode] : inputNode.childMap) {
|
|
||||||
SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
|
|
||||||
self(
|
|
||||||
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
|
|
||||||
resolveDerivedPath(store, next, evalStore_),
|
|
||||||
childNode);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
|
||||||
accumRealisations(inputDrv, inputNode);
|
|
||||||
|
|
||||||
auto info = store.queryPathInfo(outputPath);
|
|
||||||
|
|
||||||
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
|
|
||||||
}
|
|
||||||
|
|
||||||
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
|
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
|
||||||
{
|
{
|
||||||
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
|
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
#include "nix/store/realisation.hh"
|
#include "nix/store/realisation.hh"
|
||||||
#include "nix/store/store-api.hh"
|
#include "nix/store/store-api.hh"
|
||||||
#include "nix/util/closure.hh"
|
|
||||||
#include "nix/util/signature/local-keys.hh"
|
#include "nix/util/signature/local-keys.hh"
|
||||||
#include "nix/util/json-utils.hh"
|
#include "nix/util/json-utils.hh"
|
||||||
#include <nlohmann/json.hpp>
|
#include <nlohmann/json.hpp>
|
||||||
@@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const
|
|||||||
return strHash() + "!" + outputName;
|
return strHash() + "!" + outputName;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
|
|
||||||
{
|
|
||||||
std::set<Realisation> res;
|
|
||||||
Realisation::closure(store, startOutputs, res);
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
|
|
||||||
{
|
|
||||||
auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
|
|
||||||
std::set<Realisation> res;
|
|
||||||
for (auto & [currentDep, _] : current.dependentRealisations) {
|
|
||||||
if (auto currentRealisation = store.queryRealisation(currentDep))
|
|
||||||
res.insert({*currentRealisation, currentDep});
|
|
||||||
else
|
|
||||||
throw Error("Unrealised derivation '%s'", currentDep.to_string());
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
};
|
|
||||||
|
|
||||||
computeClosure<Realisation>(
|
|
||||||
startOutputs,
|
|
||||||
res,
|
|
||||||
[&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
|
|
||||||
std::promise<std::set<Realisation>> promise;
|
|
||||||
try {
|
|
||||||
auto res = getDeps(current);
|
|
||||||
promise.set_value(res);
|
|
||||||
} catch (...) {
|
|
||||||
promise.set_exception(std::current_exception());
|
|
||||||
}
|
|
||||||
return processEdges(promise);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
|
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
|
||||||
{
|
{
|
||||||
nlohmann::json serialized = Realisation{*this, key};
|
nlohmann::json serialized = Realisation{*this, key};
|
||||||
@@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const &
|
|||||||
|
|
||||||
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
|
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
|
||||||
{
|
{
|
||||||
if (outPath == other.outPath) {
|
return outPath == other.outPath;
|
||||||
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
|
|
||||||
warn(
|
|
||||||
"Encountered a realisation for '%s' with an empty set of "
|
|
||||||
"dependencies. This is likely an artifact from an older Nix. "
|
|
||||||
"I’ll try to fix the realisation if I can",
|
|
||||||
id.to_string());
|
|
||||||
return true;
|
|
||||||
} else if (dependentRealisations == other.dependentRealisations) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret)
|
|
||||||
{
|
|
||||||
// FIXME: This only builds the store-path closure, not the real realisation
|
|
||||||
// closure
|
|
||||||
StorePathSet initialStorePaths, pathsClosure;
|
|
||||||
for (auto & path : startPaths)
|
|
||||||
initialStorePaths.insert(path.path());
|
|
||||||
store.computeFSClosure(initialStorePaths, pathsClosure);
|
|
||||||
ret.insert(startPaths.begin(), startPaths.end());
|
|
||||||
ret.insert(pathsClosure.begin(), pathsClosure.end());
|
|
||||||
}
|
|
||||||
|
|
||||||
void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
|
|
||||||
{
|
|
||||||
RealisedPath::closure(store, {*this}, ret);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealisedPath::Set RealisedPath::closure(Store & store) const
|
|
||||||
{
|
|
||||||
RealisedPath::Set ret;
|
|
||||||
closure(store, ret);
|
|
||||||
return ret;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -162,27 +90,19 @@ UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & js
|
|||||||
if (auto signaturesOpt = optionalValueAt(json, "signatures"))
|
if (auto signaturesOpt = optionalValueAt(json, "signatures"))
|
||||||
signatures = *signaturesOpt;
|
signatures = *signaturesOpt;
|
||||||
|
|
||||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
|
||||||
if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations"))
|
|
||||||
for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
|
|
||||||
dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
|
|
||||||
|
|
||||||
return UnkeyedRealisation{
|
return UnkeyedRealisation{
|
||||||
.outPath = valueAt(json, "outPath"),
|
.outPath = valueAt(json, "outPath"),
|
||||||
.signatures = signatures,
|
.signatures = signatures,
|
||||||
.dependentRealisations = dependentRealisations,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
|
void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
|
||||||
{
|
{
|
||||||
auto jsonDependentRealisations = nlohmann::json::object();
|
|
||||||
for (auto & [depId, depOutPath] : r.dependentRealisations)
|
|
||||||
jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
|
|
||||||
json = {
|
json = {
|
||||||
{"outPath", r.outPath},
|
{"outPath", r.outPath},
|
||||||
{"signatures", r.signatures},
|
{"signatures", r.signatures},
|
||||||
{"dependentRealisations", jsonDependentRealisations},
|
// back-compat
|
||||||
|
{"dependentRealisations", json::object()},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -292,7 +292,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
|
|||||||
next->computeFSClosure(newPaths, closure);
|
next->computeFSClosure(newPaths, closure);
|
||||||
for (auto & path : closure)
|
for (auto & path : closure)
|
||||||
goal.addDependency(path);
|
goal.addDependency(path);
|
||||||
for (auto & real : Realisation::closure(*next, newRealisations))
|
for (auto & real : newRealisations)
|
||||||
goal.addedDrvOutputs.insert(real.id);
|
goal.addedDrvOutputs.insert(real.id);
|
||||||
|
|
||||||
return results;
|
return results;
|
||||||
|
|||||||
+18
-22
@@ -915,36 +915,21 @@ std::map<StorePath, StorePath> copyPaths(
|
|||||||
SubstituteFlag substitute)
|
SubstituteFlag substitute)
|
||||||
{
|
{
|
||||||
StorePathSet storePaths;
|
StorePathSet storePaths;
|
||||||
std::set<Realisation> toplevelRealisations;
|
std::vector<const Realisation *> realisations;
|
||||||
for (auto & path : paths) {
|
for (auto & path : paths) {
|
||||||
storePaths.insert(path.path());
|
storePaths.insert(path.path());
|
||||||
if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
|
if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
|
||||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||||
toplevelRealisations.insert(*realisation);
|
realisations.push_back(realisation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
|
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Copy the realisation closure
|
// Copy the realisations. TODO batch this
|
||||||
processGraph<Realisation>(
|
for (const auto * realisation : realisations)
|
||||||
Realisation::closure(srcStore, toplevelRealisations),
|
dstStore.registerDrvOutput(*realisation, checkSigs);
|
||||||
[&](const Realisation & current) -> std::set<Realisation> {
|
|
||||||
std::set<Realisation> children;
|
|
||||||
for (const auto & [drvOutput, _] : current.dependentRealisations) {
|
|
||||||
auto currentChild = srcStore.queryRealisation(drvOutput);
|
|
||||||
if (!currentChild)
|
|
||||||
throw Error(
|
|
||||||
"incomplete realisation closure: '%s' is a "
|
|
||||||
"dependency of '%s' but isn't registered",
|
|
||||||
drvOutput.to_string(),
|
|
||||||
current.id.to_string());
|
|
||||||
children.insert({*currentChild, drvOutput});
|
|
||||||
}
|
|
||||||
return children;
|
|
||||||
},
|
|
||||||
[&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
|
|
||||||
} catch (MissingExperimentalFeature & e) {
|
} catch (MissingExperimentalFeature & e) {
|
||||||
// Don't fail if the remote doesn't support CA derivations is it might
|
// Don't fail if the remote doesn't support CA derivations is it might
|
||||||
// not be within our control to change that, and we might still want
|
// not be within our control to change that, and we might still want
|
||||||
@@ -1055,8 +1040,19 @@ void copyClosure(
|
|||||||
if (&srcStore == &dstStore)
|
if (&srcStore == &dstStore)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
RealisedPath::Set closure;
|
StorePathSet closure0;
|
||||||
RealisedPath::closure(srcStore, paths, closure);
|
for (auto & path : paths) {
|
||||||
|
if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
|
||||||
|
closure0.insert(opaquePath->path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
StorePathSet closure1;
|
||||||
|
srcStore.computeFSClosure(closure0, closure1);
|
||||||
|
|
||||||
|
RealisedPath::Set closure = paths;
|
||||||
|
for (auto && path : closure1)
|
||||||
|
closure.insert({std::move(path)});
|
||||||
|
|
||||||
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,20 +4,11 @@
|
|||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
#include "nix/util/types.hh"
|
#include "nix/util/types.hh"
|
||||||
#include "nix/util/environment-variables.hh"
|
|
||||||
#include "nix/util/file-system.hh"
|
#include "nix/util/file-system.hh"
|
||||||
|
#include "nix/util/tests/test-data.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
/**
|
|
||||||
* The path to the unit test data directory. See the contributing guide
|
|
||||||
* in the manual for further details.
|
|
||||||
*/
|
|
||||||
static inline std::filesystem::path getUnitTestData()
|
|
||||||
{
|
|
||||||
return getEnv("_NIX_TEST_UNIT_DATA").value();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether we should update "golden masters" instead of running tests
|
* Whether we should update "golden masters" instead of running tests
|
||||||
* against them. See the contributing guide in the manual for further
|
* against them. See the contributing guide in the manual for further
|
||||||
|
|||||||
@@ -83,6 +83,31 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Specialization for when we need to do "JSON -> `ref<T>`" in one
|
||||||
|
* direction, but "`const T &` -> JSON" in the other direction.
|
||||||
|
*/
|
||||||
|
template<typename T>
|
||||||
|
void checkpointJson(CharacterizationTest & test, PathView testStem, const ref<T> & got)
|
||||||
|
{
|
||||||
|
using namespace nlohmann;
|
||||||
|
|
||||||
|
auto file = test.goldenMaster(Path{testStem} + ".json");
|
||||||
|
|
||||||
|
json gotJson = static_cast<json>(*got);
|
||||||
|
|
||||||
|
if (testAccept()) {
|
||||||
|
std::filesystem::create_directories(file.parent_path());
|
||||||
|
writeFile(file, gotJson.dump(2) + "\n");
|
||||||
|
ADD_FAILURE() << "Updating golden master " << file;
|
||||||
|
} else {
|
||||||
|
json expectedJson = json::parse(readFile(file));
|
||||||
|
ASSERT_EQ(gotJson, expectedJson);
|
||||||
|
ref<T> expected = adl_serializer<ref<T>>::from_json(expectedJson);
|
||||||
|
ASSERT_EQ(*got, *expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Mixin class for writing characterization tests for `nlohmann::json`
|
* Mixin class for writing characterization tests for `nlohmann::json`
|
||||||
* conversions for a given type.
|
* conversions for a given type.
|
||||||
|
|||||||
@@ -10,4 +10,5 @@ headers = files(
|
|||||||
'json-characterization.hh',
|
'json-characterization.hh',
|
||||||
'nix_api_util.hh',
|
'nix_api_util.hh',
|
||||||
'string_callback.hh',
|
'string_callback.hh',
|
||||||
|
'test-data.hh',
|
||||||
)
|
)
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#pragma once
|
||||||
|
///@file
|
||||||
|
|
||||||
|
#include <filesystem>
|
||||||
|
#include "nix/util/environment-variables.hh"
|
||||||
|
#include "nix/util/error.hh"
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The path to the unit test data directory. See the contributing guide
|
||||||
|
* in the manual for further details.
|
||||||
|
*/
|
||||||
|
static inline std::filesystem::path getUnitTestData()
|
||||||
|
{
|
||||||
|
auto data = getEnv("_NIX_TEST_UNIT_DATA");
|
||||||
|
if (!data)
|
||||||
|
throw Error(
|
||||||
|
"_NIX_TEST_UNIT_DATA environment variable is not set. "
|
||||||
|
"Recommendation: use meson, example: 'meson test -C build --gdb'");
|
||||||
|
return std::filesystem::path(*data);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
@@ -20,10 +20,8 @@ TEST(closure, correctClosure)
|
|||||||
set<string> aClosure;
|
set<string> aClosure;
|
||||||
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
|
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
|
||||||
computeClosure<string>(
|
computeClosure<string>(
|
||||||
{"A"}, aClosure, [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
|
{"A"}, aClosure, [&](const std::string & currentNode) -> asio::awaitable<std::set<std::string>> {
|
||||||
promise<set<string>> promisedNodes;
|
co_return testGraph[currentNode];
|
||||||
promisedNodes.set_value(testGraph[currentNode]);
|
|
||||||
processEdges(promisedNodes);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
ASSERT_EQ(aClosure, expectedClosure);
|
ASSERT_EQ(aClosure, expectedClosure);
|
||||||
@@ -37,31 +35,7 @@ TEST(closure, properlyHandlesDirectExceptions)
|
|||||||
set<string> aClosure;
|
set<string> aClosure;
|
||||||
EXPECT_THROW(
|
EXPECT_THROW(
|
||||||
computeClosure<string>(
|
computeClosure<string>(
|
||||||
{"A"},
|
{"A"}, aClosure, [&](const std::string &) -> asio::awaitable<std::set<std::string>> { throw TestExn(); }),
|
||||||
aClosure,
|
|
||||||
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) { throw TestExn(); }),
|
|
||||||
TestExn);
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(closure, properlyHandlesExceptionsInPromise)
|
|
||||||
{
|
|
||||||
struct TestExn
|
|
||||||
{};
|
|
||||||
|
|
||||||
set<string> aClosure;
|
|
||||||
EXPECT_THROW(
|
|
||||||
computeClosure<string>(
|
|
||||||
{"A"},
|
|
||||||
aClosure,
|
|
||||||
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
|
|
||||||
promise<set<string>> promise;
|
|
||||||
try {
|
|
||||||
throw TestExn();
|
|
||||||
} catch (...) {
|
|
||||||
promise.set_exception(std::current_exception());
|
|
||||||
}
|
|
||||||
processEdges(promise);
|
|
||||||
}),
|
|
||||||
TestExn);
|
TestExn);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -388,14 +388,13 @@ TEST(openFileEnsureBeneathNoSymlinks, works)
|
|||||||
TEST(createAnonymousTempFile, works)
|
TEST(createAnonymousTempFile, works)
|
||||||
{
|
{
|
||||||
auto fd = createAnonymousTempFile();
|
auto fd = createAnonymousTempFile();
|
||||||
auto fd_ = fromDescriptorReadOnly(fd.get());
|
|
||||||
writeFull(fd.get(), "test");
|
writeFull(fd.get(), "test");
|
||||||
lseek(fd_, 0, SEEK_SET);
|
lseek(fd.get(), 0, SEEK_SET);
|
||||||
FdSource source{fd.get()};
|
FdSource source{fd.get()};
|
||||||
EXPECT_EQ(source.drain(), "test");
|
EXPECT_EQ(source.drain(), "test");
|
||||||
lseek(fd_, 0, SEEK_END);
|
lseek(fd.get(), 0, SEEK_END);
|
||||||
writeFull(fd.get(), "test");
|
writeFull(fd.get(), "test");
|
||||||
lseek(fd_, 0, SEEK_SET);
|
lseek(fd.get(), 0, SEEK_SET);
|
||||||
EXPECT_EQ(source.drain(), "testtest");
|
EXPECT_EQ(source.drain(), "testtest");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -406,9 +405,8 @@ TEST(createAnonymousTempFile, works)
|
|||||||
TEST(FdSource, restartWorks)
|
TEST(FdSource, restartWorks)
|
||||||
{
|
{
|
||||||
auto fd = createAnonymousTempFile();
|
auto fd = createAnonymousTempFile();
|
||||||
auto fd_ = fromDescriptorReadOnly(fd.get());
|
|
||||||
writeFull(fd.get(), "hello world");
|
writeFull(fd.get(), "hello world");
|
||||||
lseek(fd_, 0, SEEK_SET);
|
lseek(fd.get(), 0, SEEK_SET);
|
||||||
FdSource source{fd.get()};
|
FdSource source{fd.get()};
|
||||||
EXPECT_EQ(source.drain(), "hello world");
|
EXPECT_EQ(source.drain(), "hello world");
|
||||||
source.restart();
|
source.restart();
|
||||||
@@ -416,4 +414,11 @@ TEST(FdSource, restartWorks)
|
|||||||
EXPECT_EQ(source.drain(), "");
|
EXPECT_EQ(source.drain(), "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST(createTempDir, works)
|
||||||
|
{
|
||||||
|
auto tmpDir = createTempDir();
|
||||||
|
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
|
||||||
|
ASSERT_TRUE(std::filesystem::is_directory(tmpDir));
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -33,6 +33,9 @@ deps_private += rapidcheck
|
|||||||
gtest = dependency('gtest', main : true)
|
gtest = dependency('gtest', main : true)
|
||||||
deps_private += gtest
|
deps_private += gtest
|
||||||
|
|
||||||
|
gmock = dependency('gmock')
|
||||||
|
deps_private += gmock
|
||||||
|
|
||||||
configdata = configuration_data()
|
configdata = configuration_data()
|
||||||
configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
|
configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
|
||||||
|
|
||||||
@@ -72,6 +75,7 @@ sources = files(
|
|||||||
'position.cc',
|
'position.cc',
|
||||||
'processes.cc',
|
'processes.cc',
|
||||||
'sort.cc',
|
'sort.cc',
|
||||||
|
'source-accessor.cc',
|
||||||
'spawn.cc',
|
'spawn.cc',
|
||||||
'strings.cc',
|
'strings.cc',
|
||||||
'suggestions.cc',
|
'suggestions.cc',
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
#include "nix/util/fs-sink.hh"
|
||||||
|
#include "nix/util/file-system.hh"
|
||||||
|
#include "nix/util/processes.hh"
|
||||||
|
|
||||||
|
#include <gtest/gtest.h>
|
||||||
|
#include <gmock/gmock.h>
|
||||||
|
#include <rapidcheck/gtest.h>
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
MATCHER_P2(HasContents, path, expected, "")
|
||||||
|
{
|
||||||
|
auto stat = arg->maybeLstat(path);
|
||||||
|
if (!stat) {
|
||||||
|
*result_listener << arg->showPath(path) << " does not exist";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (stat->type != SourceAccessor::tRegular) {
|
||||||
|
*result_listener << arg->showPath(path) << " is not a regular file";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
auto actual = arg->readFile(path);
|
||||||
|
if (actual != expected) {
|
||||||
|
*result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
MATCHER_P2(HasSymlink, path, target, "")
|
||||||
|
{
|
||||||
|
auto stat = arg->maybeLstat(path);
|
||||||
|
if (!stat) {
|
||||||
|
*result_listener << arg->showPath(path) << " does not exist";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (stat->type != SourceAccessor::tSymlink) {
|
||||||
|
*result_listener << arg->showPath(path) << " is not a symlink";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
auto actual = arg->readLink(path);
|
||||||
|
if (actual != target) {
|
||||||
|
*result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
MATCHER_P2(HasDirectory, path, dirents, "")
|
||||||
|
{
|
||||||
|
auto stat = arg->maybeLstat(path);
|
||||||
|
if (!stat) {
|
||||||
|
*result_listener << arg->showPath(path) << " does not exist";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (stat->type != SourceAccessor::tDirectory) {
|
||||||
|
*result_listener << arg->showPath(path) << " is not a directory";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
auto actual = arg->readDirectory(path);
|
||||||
|
std::set<std::string> actualKeys, expectedKeys(dirents.begin(), dirents.end());
|
||||||
|
for (auto & [k, _] : actual)
|
||||||
|
actualKeys.insert(k);
|
||||||
|
if (actualKeys != expectedKeys) {
|
||||||
|
*result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
class FSSourceAccessorTest : public ::testing::Test
|
||||||
|
{
|
||||||
|
protected:
|
||||||
|
std::filesystem::path tmpDir;
|
||||||
|
std::unique_ptr<nix::AutoDelete> delTmpDir;
|
||||||
|
|
||||||
|
void SetUp() override
|
||||||
|
{
|
||||||
|
tmpDir = nix::createTempDir();
|
||||||
|
delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
void TearDown() override
|
||||||
|
{
|
||||||
|
delTmpDir.reset();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(FSSourceAccessorTest, works)
|
||||||
|
{
|
||||||
|
{
|
||||||
|
RestoreSink sink(false);
|
||||||
|
sink.dstPath = tmpDir;
|
||||||
|
#ifndef _WIN32
|
||||||
|
sink.dirFd = openDirectory(tmpDir);
|
||||||
|
#endif
|
||||||
|
sink.createDirectory(CanonPath("subdir"));
|
||||||
|
sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); });
|
||||||
|
sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); });
|
||||||
|
sink.createSymlink(CanonPath("rootlink"), "target");
|
||||||
|
sink.createDirectory(CanonPath("a"));
|
||||||
|
sink.createSymlink(CanonPath("a/dirlink"), "../subdir");
|
||||||
|
}
|
||||||
|
|
||||||
|
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), HasContents(CanonPath::root, "content1"));
|
||||||
|
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), HasSymlink(CanonPath::root, "target"));
|
||||||
|
EXPECT_THAT(
|
||||||
|
makeFSSourceAccessor(tmpDir),
|
||||||
|
HasDirectory(CanonPath::root, std::set<std::string>{"file1", "subdir", "rootlink", "a"}));
|
||||||
|
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "subdir"), HasDirectory(CanonPath::root, std::set<std::string>{"file2"}));
|
||||||
|
|
||||||
|
{
|
||||||
|
auto accessor = makeFSSourceAccessor(tmpDir);
|
||||||
|
EXPECT_THAT(accessor, HasContents(CanonPath("file1"), "content1"));
|
||||||
|
EXPECT_THAT(accessor, HasContents(CanonPath("subdir/file2"), "content2"));
|
||||||
|
|
||||||
|
EXPECT_TRUE(accessor->pathExists(CanonPath("file1")));
|
||||||
|
EXPECT_FALSE(accessor->pathExists(CanonPath("nonexistent")));
|
||||||
|
|
||||||
|
EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||||
|
EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||||
|
EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed);
|
||||||
|
EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent");
|
||||||
|
EXPECT_FALSE(accessor->maybeLstat(CanonPath::root));
|
||||||
|
EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
auto accessor = makeFSSourceAccessor(tmpDir, true);
|
||||||
|
EXPECT_EQ(accessor->getLastModified(), 0);
|
||||||
|
accessor->maybeLstat(CanonPath("file1"));
|
||||||
|
EXPECT_GT(accessor->getLastModified(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
+77
-14
@@ -69,24 +69,54 @@ struct ArchiveDecompressionSource : Source
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* These strings are a part of the public API in store parameters and such. Do not change!
|
||||||
|
Happens to match enum names. */
|
||||||
|
#define NIX_FOR_EACH_LA_ALGO(MACRO) \
|
||||||
|
MACRO(bzip2) \
|
||||||
|
MACRO(compress) \
|
||||||
|
MACRO(grzip) \
|
||||||
|
MACRO(gzip) \
|
||||||
|
MACRO(lrzip) \
|
||||||
|
MACRO(lz4) \
|
||||||
|
MACRO(lzip) \
|
||||||
|
MACRO(lzma) \
|
||||||
|
MACRO(lzop) \
|
||||||
|
MACRO(xz) \
|
||||||
|
MACRO(zstd)
|
||||||
|
|
||||||
struct ArchiveCompressionSink : CompressionSink
|
struct ArchiveCompressionSink : CompressionSink
|
||||||
{
|
{
|
||||||
Sink & nextSink;
|
Sink & nextSink;
|
||||||
struct archive * archive;
|
struct archive * archive;
|
||||||
|
|
||||||
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
|
ArchiveCompressionSink(
|
||||||
|
Sink & nextSink, CompressionAlgo method, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
|
||||||
: nextSink(nextSink)
|
: nextSink(nextSink)
|
||||||
{
|
{
|
||||||
archive = archive_write_new();
|
archive = archive_write_new();
|
||||||
if (!archive)
|
if (!archive)
|
||||||
throw Error("failed to initialize libarchive");
|
throw Error("failed to initialize libarchive");
|
||||||
check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
|
|
||||||
|
auto [addFilter, format] = [method]() -> std::pair<int (*)(struct archive *), const char *> {
|
||||||
|
switch (method) {
|
||||||
|
case CompressionAlgo::none:
|
||||||
|
case CompressionAlgo::brotli:
|
||||||
|
unreachable();
|
||||||
|
#define NIX_DEF_LA_ALGO_CASE(algo) \
|
||||||
|
case CompressionAlgo::algo: \
|
||||||
|
return {archive_write_add_filter_##algo, #algo};
|
||||||
|
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
|
||||||
|
#undef NIX_DEF_LA_ALGO_CASE
|
||||||
|
}
|
||||||
|
unreachable();
|
||||||
|
}();
|
||||||
|
|
||||||
|
check(addFilter(archive), "couldn't initialize compression (%s)");
|
||||||
check(archive_write_set_format_raw(archive));
|
check(archive_write_set_format_raw(archive));
|
||||||
if (parallel)
|
if (parallel)
|
||||||
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
|
check(archive_write_set_filter_option(archive, format, "threads", "0"));
|
||||||
if (level != COMPRESSION_LEVEL_DEFAULT)
|
if (level != COMPRESSION_LEVEL_DEFAULT)
|
||||||
check(archive_write_set_filter_option(
|
check(archive_write_set_filter_option(archive, format, "compression-level", std::to_string(level).c_str()));
|
||||||
archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
|
|
||||||
// disable internal buffering
|
// disable internal buffering
|
||||||
check(archive_write_set_bytes_per_block(archive, 0));
|
check(archive_write_set_bytes_per_block(archive, 0));
|
||||||
// disable output padding
|
// disable output padding
|
||||||
@@ -289,19 +319,52 @@ struct BrotliCompressionSink : ChunkedCompressionSink
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* Parses a *compression* method into the corresponding enum. This is only used
|
||||||
|
in the *compression* case and user interface. Content-Encoding should not use
|
||||||
|
these. */
|
||||||
|
static CompressionAlgo parseNixCompressionAlgoString(std::string_view method)
|
||||||
|
{
|
||||||
|
static const std::unordered_map<std::string_view, CompressionAlgo> lookupTable = {
|
||||||
|
{"none", CompressionAlgo::none},
|
||||||
|
{"br", CompressionAlgo::brotli},
|
||||||
|
#define NIX_DEF_LA_ALGO_NAME(algo) {#algo, CompressionAlgo::algo},
|
||||||
|
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_NAME)
|
||||||
|
#undef NIX_DEF_LA_ALGO_NAME
|
||||||
|
};
|
||||||
|
|
||||||
|
if (auto it = lookupTable.find(method); it != lookupTable.end())
|
||||||
|
return it->second;
|
||||||
|
|
||||||
|
static const StringSet allNames = [&]() {
|
||||||
|
StringSet res;
|
||||||
|
for (auto & [name, _] : lookupTable)
|
||||||
|
res.emplace(name);
|
||||||
|
return res;
|
||||||
|
}();
|
||||||
|
|
||||||
|
throw UnknownCompressionMethod(
|
||||||
|
Suggestions::bestMatches(allNames, method), "unknown compression method '%s'", method);
|
||||||
|
}
|
||||||
|
|
||||||
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
|
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
|
||||||
{
|
{
|
||||||
std::vector<std::string> la_supports = {
|
return makeCompressionSink(parseNixCompressionAlgoString(method), nextSink, parallel, level);
|
||||||
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
|
}
|
||||||
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
|
|
||||||
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
|
ref<CompressionSink> makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level)
|
||||||
}
|
{
|
||||||
if (method == "none")
|
switch (method) {
|
||||||
|
case CompressionAlgo::none:
|
||||||
return make_ref<NoneSink>(nextSink);
|
return make_ref<NoneSink>(nextSink);
|
||||||
else if (method == "br")
|
case CompressionAlgo::brotli:
|
||||||
return make_ref<BrotliCompressionSink>(nextSink);
|
return make_ref<BrotliCompressionSink>(nextSink);
|
||||||
else
|
/* Everything else is supported via libarchive. */
|
||||||
throw UnknownCompressionMethod("unknown compression method '%s'", method);
|
#define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo:
|
||||||
|
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
|
||||||
|
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
|
||||||
|
#undef NIX_DEF_LA_ALGO_CASE
|
||||||
|
}
|
||||||
|
unreachable();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)
|
std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)
|
||||||
|
|||||||
+16
-166
@@ -115,9 +115,6 @@ Path canonPath(PathView path, bool resolveSymlinks)
|
|||||||
if (!isAbsolute(path))
|
if (!isAbsolute(path))
|
||||||
throw Error("not an absolute path: '%1%'", path);
|
throw Error("not an absolute path: '%1%'", path);
|
||||||
|
|
||||||
// For Windows
|
|
||||||
auto rootName = std::filesystem::path{path}.root_name();
|
|
||||||
|
|
||||||
/* This just exists because we cannot set the target of `remaining`
|
/* This just exists because we cannot set the target of `remaining`
|
||||||
(the callback parameter) directly to a newly-constructed string,
|
(the callback parameter) directly to a newly-constructed string,
|
||||||
since it is `std::string_view`. */
|
since it is `std::string_view`. */
|
||||||
@@ -147,8 +144,6 @@ Path canonPath(PathView path, bool resolveSymlinks)
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!rootName.empty())
|
|
||||||
ret = rootName.string() + std::move(ret);
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -380,14 +375,6 @@ void syncParent(const Path & path)
|
|||||||
fd.fsync();
|
fd.fsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef __FreeBSD__
|
|
||||||
# define MOUNTEDPATHS_PARAM , std::set<Path> & mountedPaths
|
|
||||||
# define MOUNTEDPATHS_ARG , mountedPaths
|
|
||||||
#else
|
|
||||||
# define MOUNTEDPATHS_PARAM
|
|
||||||
# define MOUNTEDPATHS_ARG
|
|
||||||
#endif
|
|
||||||
|
|
||||||
void recursiveSync(const Path & path)
|
void recursiveSync(const Path & path)
|
||||||
{
|
{
|
||||||
/* If it's a file or symlink, just fsync and return. */
|
/* If it's a file or symlink, just fsync and return. */
|
||||||
@@ -432,129 +419,6 @@ void recursiveSync(const Path & path)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void _deletePath(
|
|
||||||
Descriptor parentfd,
|
|
||||||
const std::filesystem::path & path,
|
|
||||||
uint64_t & bytesFreed,
|
|
||||||
std::exception_ptr & ex MOUNTEDPATHS_PARAM)
|
|
||||||
{
|
|
||||||
#ifndef _WIN32
|
|
||||||
checkInterrupt();
|
|
||||||
|
|
||||||
# ifdef __FreeBSD__
|
|
||||||
// In case of emergency (unmount fails for some reason) not recurse into mountpoints.
|
|
||||||
// This prevents us from tearing up the nullfs-mounted nix store.
|
|
||||||
if (mountedPaths.find(path) != mountedPaths.end()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
# endif
|
|
||||||
|
|
||||||
std::string name(path.filename());
|
|
||||||
assert(name != "." && name != ".." && !name.empty());
|
|
||||||
|
|
||||||
struct stat st;
|
|
||||||
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
|
|
||||||
if (errno == ENOENT)
|
|
||||||
return;
|
|
||||||
throw SysError("getting status of %1%", path);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!S_ISDIR(st.st_mode)) {
|
|
||||||
/* We are about to delete a file. Will it likely free space? */
|
|
||||||
|
|
||||||
switch (st.st_nlink) {
|
|
||||||
/* Yes: last link. */
|
|
||||||
case 1:
|
|
||||||
bytesFreed += st.st_size;
|
|
||||||
break;
|
|
||||||
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
|
|
||||||
was performed. Instead of checking for real let's assume
|
|
||||||
it's an optimised file and space will be freed.
|
|
||||||
|
|
||||||
In worst case we will double count on freed space for files
|
|
||||||
with exactly two hardlinks for unoptimised packages.
|
|
||||||
*/
|
|
||||||
case 2:
|
|
||||||
bytesFreed += st.st_size;
|
|
||||||
break;
|
|
||||||
/* No: 3+ links. */
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
|
||||||
/* Make the directory accessible. */
|
|
||||||
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
|
|
||||||
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
|
|
||||||
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
|
|
||||||
throw SysError("chmod %1%", path);
|
|
||||||
}
|
|
||||||
|
|
||||||
int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
|
|
||||||
if (fd == -1)
|
|
||||||
throw SysError("opening directory %1%", path);
|
|
||||||
AutoCloseDir dir(fdopendir(fd));
|
|
||||||
if (!dir)
|
|
||||||
throw SysError("opening directory %1%", path);
|
|
||||||
|
|
||||||
struct dirent * dirent;
|
|
||||||
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
|
||||||
checkInterrupt();
|
|
||||||
std::string childName = dirent->d_name;
|
|
||||||
if (childName == "." || childName == "..")
|
|
||||||
continue;
|
|
||||||
_deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG);
|
|
||||||
}
|
|
||||||
if (errno)
|
|
||||||
throw SysError("reading directory %1%", path);
|
|
||||||
}
|
|
||||||
|
|
||||||
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
|
|
||||||
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
|
|
||||||
if (errno == ENOENT)
|
|
||||||
return;
|
|
||||||
try {
|
|
||||||
throw SysError("cannot unlink %1%", path);
|
|
||||||
} catch (...) {
|
|
||||||
if (!ex)
|
|
||||||
ex = std::current_exception();
|
|
||||||
else
|
|
||||||
ignoreExceptionExceptInterrupt();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
// TODO implement
|
|
||||||
throw UnimplementedError("_deletePath");
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM)
|
|
||||||
{
|
|
||||||
assert(path.is_absolute());
|
|
||||||
assert(path.parent_path() != path);
|
|
||||||
|
|
||||||
AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
|
|
||||||
if (!dirfd) {
|
|
||||||
if (errno == ENOENT)
|
|
||||||
return;
|
|
||||||
throw SysError("opening directory %s", path.parent_path());
|
|
||||||
}
|
|
||||||
|
|
||||||
std::exception_ptr ex;
|
|
||||||
|
|
||||||
_deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG);
|
|
||||||
|
|
||||||
if (ex)
|
|
||||||
std::rethrow_exception(ex);
|
|
||||||
}
|
|
||||||
|
|
||||||
void deletePath(const std::filesystem::path & path)
|
|
||||||
{
|
|
||||||
uint64_t dummy;
|
|
||||||
deletePath(path, dummy);
|
|
||||||
}
|
|
||||||
|
|
||||||
void createDir(const Path & path, mode_t mode)
|
void createDir(const Path & path, mode_t mode)
|
||||||
{
|
{
|
||||||
if (mkdir(
|
if (mkdir(
|
||||||
@@ -577,25 +441,6 @@ void createDirs(const std::filesystem::path & path)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
|
|
||||||
{
|
|
||||||
// Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
|
|
||||||
#ifdef __FreeBSD__
|
|
||||||
std::set<Path> mountedPaths;
|
|
||||||
struct statfs * mntbuf;
|
|
||||||
int count;
|
|
||||||
if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
|
|
||||||
throw SysError("getmntinfo");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
mountedPaths.emplace(mntbuf[i].f_mntonname);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
bytesFreed = 0;
|
|
||||||
_deletePath(path, bytesFreed MOUNTEDPATHS_ARG);
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
AutoDelete::AutoDelete()
|
AutoDelete::AutoDelete()
|
||||||
@@ -672,11 +517,6 @@ void AutoUnmount::cancel()
|
|||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
std::filesystem::path defaultTempDir()
|
|
||||||
{
|
|
||||||
return getEnvNonEmpty("TMPDIR").value_or("/tmp");
|
|
||||||
}
|
|
||||||
|
|
||||||
std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode)
|
std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode)
|
||||||
{
|
{
|
||||||
while (1) {
|
while (1) {
|
||||||
@@ -713,17 +553,27 @@ AutoCloseFD createAnonymousTempFile()
|
|||||||
{
|
{
|
||||||
AutoCloseFD fd;
|
AutoCloseFD fd;
|
||||||
#ifdef O_TMPFILE
|
#ifdef O_TMPFILE
|
||||||
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR, S_IWUSR | S_IRUSR);
|
static std::atomic_flag tmpfileUnsupported{};
|
||||||
if (!fd)
|
if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ {
|
||||||
throw SysError("creating anonymous temporary file");
|
/* Use O_EXCL, because the file is never supposed to be linked into filesystem. */
|
||||||
#else
|
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR | O_EXCL, S_IWUSR | S_IRUSR);
|
||||||
|
if (!fd) {
|
||||||
|
/* Not supported by the filesystem or the kernel. */
|
||||||
|
if (errno == EOPNOTSUPP || errno == EISDIR)
|
||||||
|
tmpfileUnsupported.test_and_set(); /* Set flag and fall through to createTempFile. */
|
||||||
|
else
|
||||||
|
throw SysError("creating anonymous temporary file");
|
||||||
|
} else {
|
||||||
|
return fd; /* Successfully created. */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
auto [fd2, path] = createTempFile("nix-anonymous");
|
auto [fd2, path] = createTempFile("nix-anonymous");
|
||||||
if (!fd2)
|
if (!fd2)
|
||||||
throw SysError("creating temporary file '%s'", path);
|
throw SysError("creating temporary file '%s'", path);
|
||||||
fd = std::move(fd2);
|
fd = std::move(fd2);
|
||||||
# ifndef _WIN32
|
#ifndef _WIN32
|
||||||
unlink(requireCString(path)); /* We only care about the file descriptor. */
|
unlink(requireCString(path)); /* We only care about the file descriptor. */
|
||||||
# endif
|
|
||||||
#endif
|
#endif
|
||||||
return fd;
|
return fd;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,24 @@ class AutoRemoveJail
|
|||||||
bool del;
|
bool del;
|
||||||
public:
|
public:
|
||||||
AutoRemoveJail(int jid);
|
AutoRemoveJail(int jid);
|
||||||
|
AutoRemoveJail(const AutoRemoveJail &) = delete;
|
||||||
|
AutoRemoveJail & operator=(const AutoRemoveJail &) = delete;
|
||||||
|
|
||||||
|
AutoRemoveJail(AutoRemoveJail && other) noexcept
|
||||||
|
: jid(other.jid)
|
||||||
|
, del(other.del)
|
||||||
|
{
|
||||||
|
other.cancel();
|
||||||
|
}
|
||||||
|
|
||||||
|
AutoRemoveJail & operator=(AutoRemoveJail && other) noexcept
|
||||||
|
{
|
||||||
|
jid = other.jid;
|
||||||
|
del = other.del;
|
||||||
|
other.cancel();
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
AutoRemoveJail();
|
AutoRemoveJail();
|
||||||
~AutoRemoveJail();
|
~AutoRemoveJail();
|
||||||
void cancel();
|
void cancel();
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
#pragma once
|
||||||
|
///@file
|
||||||
|
|
||||||
|
#include "nix/util/callback.hh"
|
||||||
|
#include "nix/util/ref.hh"
|
||||||
|
#include "nix/util/signals.hh"
|
||||||
|
|
||||||
|
#include <boost/asio/io_context.hpp>
|
||||||
|
#include <boost/asio/post.hpp>
|
||||||
|
#include <boost/asio/awaitable.hpp>
|
||||||
|
#include <boost/asio/use_awaitable.hpp>
|
||||||
|
#include <boost/asio/associated_cancellation_slot.hpp>
|
||||||
|
|
||||||
|
#include <concepts>
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
namespace asio = boost::asio;
|
||||||
|
|
||||||
|
template<typename T, std::invocable<Callback<T>> F, typename CompletionToken>
|
||||||
|
auto callbackToAwaitable(F && initiate, CompletionToken && token)
|
||||||
|
{
|
||||||
|
return asio::async_initiate<CompletionToken, void(std::future<T>)>(
|
||||||
|
[initiate = std::forward<F>(initiate)](auto handler) mutable {
|
||||||
|
auto executor = asio::get_associated_executor(handler);
|
||||||
|
auto done = std::make_shared<std::atomic<bool>>(false);
|
||||||
|
auto h = std::make_shared<decltype(handler)>(std::move(handler));
|
||||||
|
|
||||||
|
if (auto slot = asio::get_associated_cancellation_slot(*h); slot.is_connected()) {
|
||||||
|
std::weak_ptr wh = h; /* To handle the cyclic ownership. */
|
||||||
|
std::weak_ptr wdone = done;
|
||||||
|
slot.assign([executor, wh, wdone](asio::cancellation_type /*don't care*/) {
|
||||||
|
auto h = wh.lock();
|
||||||
|
auto done = wdone.lock();
|
||||||
|
if (!h || !done || done->exchange(true))
|
||||||
|
return; /* Gracefully die. */
|
||||||
|
/* Doesn't need to be kept alive for get_future() since it shares the ownership. */
|
||||||
|
std::promise<T> p;
|
||||||
|
p.set_exception(std::make_exception_ptr(Interrupted("interrupted by user")));
|
||||||
|
asio::post(executor, [h, fut = p.get_future()]() mutable { std::move (*h)(std::move(fut)); });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
initiate(Callback<T>([executor, done, h](std::future<T> fut) mutable {
|
||||||
|
if (done->exchange(true))
|
||||||
|
/* Early return for cooperative cancellation. The callback has been caller
|
||||||
|
later than we've been cancelled. In practice we'll get an error, the handler
|
||||||
|
has already been posted by the cancellation handler. */
|
||||||
|
return;
|
||||||
|
asio::post(executor, [h, fut = std::move(fut)]() mutable { std::move (*h)(std::move(fut)); });
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
std::forward<CompletionToken>(token));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a completion handler callback into a stackless coroutine. The
|
||||||
|
* callback can be invoked on any thread and the completion handler will be
|
||||||
|
* marshalled to the coroutines executer.
|
||||||
|
*/
|
||||||
|
template<typename T, std::invocable<Callback<T>> F>
|
||||||
|
asio::awaitable<T> callbackToAwaitable(F && initiate)
|
||||||
|
{
|
||||||
|
auto fut = co_await callbackToAwaitable<T>(std::forward<F>(initiate), asio::use_awaitable);
|
||||||
|
co_return fut.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace nix
|
||||||
@@ -1,73 +1,167 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
///@file
|
///@file
|
||||||
|
|
||||||
#include <set>
|
#include "nix/util/async.hh"
|
||||||
#include <future>
|
#include "nix/util/ref.hh"
|
||||||
#include "nix/util/sync.hh"
|
|
||||||
|
|
||||||
using std::set;
|
#include <boost/asio/io_context.hpp>
|
||||||
|
#include <boost/asio/post.hpp>
|
||||||
|
#include <boost/asio/strand.hpp>
|
||||||
|
#include <boost/asio/detached.hpp>
|
||||||
|
#include <boost/asio/awaitable.hpp>
|
||||||
|
#include <boost/asio/co_spawn.hpp>
|
||||||
|
#include <boost/asio/bind_executor.hpp>
|
||||||
|
#include <boost/asio/executor_work_guard.hpp>
|
||||||
|
|
||||||
|
#include <queue>
|
||||||
|
#include <set>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
template<typename T>
|
template<typename T>
|
||||||
using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
|
using GetEdgesAsync = std::function<asio::awaitable<std::set<T>>(const T & elt)>;
|
||||||
|
|
||||||
template<typename T>
|
template<typename T, typename CompletionToken>
|
||||||
void computeClosure(const set<T> startElts, set<T> & res, GetEdgesAsync<T> getEdgesAsync)
|
auto computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges, CompletionToken && token)
|
||||||
{
|
{
|
||||||
struct State
|
auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) {
|
||||||
{
|
auto executor = asio::make_strand(asio::get_associated_executor(handler));
|
||||||
size_t pending;
|
|
||||||
set<T> & res;
|
|
||||||
std::exception_ptr exc;
|
|
||||||
};
|
|
||||||
|
|
||||||
Sync<State> state_(State{0, res, 0});
|
/* Hand-rolled dynamic async graph traversal. ASIO/Cobalt and standard
|
||||||
|
* C++ will get channels and task groups at some point, but this will
|
||||||
std::condition_variable done;
|
* have to suffice for now. */
|
||||||
|
struct State : std::enable_shared_from_this<State>
|
||||||
auto enqueue = [&](this auto & enqueue, const T & current) -> void {
|
|
||||||
{
|
{
|
||||||
auto state(state_.lock());
|
decltype(executor) executor;
|
||||||
if (state->exc)
|
decltype(getEdges) getEdges;
|
||||||
return;
|
decltype(handler) handler;
|
||||||
if (!state->res.insert(current).second)
|
std::set<T> & res;
|
||||||
return;
|
/**
|
||||||
state->pending++;
|
* Needed to keep the ctx.run() alive because actual work might be happening on another thread
|
||||||
|
* (like with FileTransfer case).
|
||||||
|
*/
|
||||||
|
asio::executor_work_guard<decltype(State::executor)> workGuard;
|
||||||
|
std::size_t pending = 0;
|
||||||
|
/**
|
||||||
|
* Whether the completion handler has been called.
|
||||||
|
*/
|
||||||
|
bool done = false;
|
||||||
|
/**
|
||||||
|
* Amount of coroutines currently in flight.
|
||||||
|
*/
|
||||||
|
std::size_t inFlight = 0;
|
||||||
|
/**
|
||||||
|
* Maximum number of concurrent coroutines. Implements primitive rate limiting.
|
||||||
|
*/
|
||||||
|
std::size_t maxConcurrent = 64;
|
||||||
|
/**
|
||||||
|
* Nodes to handle next.
|
||||||
|
*/
|
||||||
|
std::queue<T> todo;
|
||||||
|
|
||||||
|
State(
|
||||||
|
decltype(executor) executor_, decltype(getEdges) getEdges, decltype(handler) handler, std::set<T> & res)
|
||||||
|
: executor(executor_)
|
||||||
|
, getEdges(std::move(getEdges))
|
||||||
|
, handler(std::move(handler))
|
||||||
|
, res(res)
|
||||||
|
, workGuard(asio::make_work_guard(executor_))
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
void complete(std::exception_ptr ex)
|
||||||
|
{
|
||||||
|
if (std::exchange(done, true))
|
||||||
|
return;
|
||||||
|
workGuard.reset(); /* We are done and we can release the lock. */
|
||||||
|
asio::post(executor, [state = this->shared_from_this(), ex] { state->handler(ex); });
|
||||||
|
}
|
||||||
|
|
||||||
|
void enqueue(const std::set<T> & elts)
|
||||||
|
{
|
||||||
|
for (const auto & elt : elts)
|
||||||
|
enqueue(elt);
|
||||||
|
}
|
||||||
|
|
||||||
|
void spawnWorker(const T & elt)
|
||||||
|
{
|
||||||
|
++inFlight;
|
||||||
|
auto state = this->shared_from_this();
|
||||||
|
asio::post(executor, [state = this->shared_from_this(), elt = std::move(elt)] {
|
||||||
|
asio::co_spawn(
|
||||||
|
state->executor,
|
||||||
|
[state, elt]() -> asio::awaitable<void> {
|
||||||
|
try {
|
||||||
|
state->enqueue(co_await state->getEdges(elt));
|
||||||
|
} catch (...) {
|
||||||
|
state->complete(std::current_exception());
|
||||||
|
}
|
||||||
|
state->onWorkDone();
|
||||||
|
},
|
||||||
|
asio::detached);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
void onWorkDone()
|
||||||
|
{
|
||||||
|
--inFlight;
|
||||||
|
--pending;
|
||||||
|
|
||||||
|
if (!todo.empty()) {
|
||||||
|
auto next = std::move(todo.front());
|
||||||
|
todo.pop();
|
||||||
|
asio::post(executor, [state = this->shared_from_this(), next = std::move(next)]() mutable {
|
||||||
|
state->spawnWorker(std::move(next));
|
||||||
|
});
|
||||||
|
} else if (pending == 0) {
|
||||||
|
complete(std::exception_ptr{});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void enqueue(const T & elt)
|
||||||
|
{
|
||||||
|
if (done)
|
||||||
|
return;
|
||||||
|
|
||||||
|
if (!res.insert(elt).second)
|
||||||
|
return;
|
||||||
|
|
||||||
|
++pending;
|
||||||
|
|
||||||
|
if (inFlight < maxConcurrent) {
|
||||||
|
spawnWorker(elt);
|
||||||
|
} else {
|
||||||
|
todo.push(elt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
auto state = make_ref<State>(executor, std::move(getEdges), std::move(handler), res);
|
||||||
|
if (startElts.empty()) {
|
||||||
|
/* No work to do. */
|
||||||
|
state->complete(std::exception_ptr{});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
|
asio::post(executor, [state, startElts = std::move(startElts)] { state->enqueue(startElts); });
|
||||||
try {
|
|
||||||
auto children = prom.get_future().get();
|
|
||||||
for (auto & child : children)
|
|
||||||
enqueue(child);
|
|
||||||
{
|
|
||||||
auto state(state_.lock());
|
|
||||||
assert(state->pending);
|
|
||||||
if (!--state->pending)
|
|
||||||
done.notify_one();
|
|
||||||
}
|
|
||||||
} catch (...) {
|
|
||||||
auto state(state_.lock());
|
|
||||||
if (!state->exc)
|
|
||||||
state->exc = std::current_exception();
|
|
||||||
assert(state->pending);
|
|
||||||
if (!--state->pending)
|
|
||||||
done.notify_one();
|
|
||||||
};
|
|
||||||
});
|
|
||||||
};
|
};
|
||||||
|
|
||||||
for (auto & startElt : startElts)
|
return asio::async_initiate<CompletionToken, void(std::exception_ptr)>(std::move(initiator), token);
|
||||||
enqueue(startElt);
|
}
|
||||||
|
|
||||||
{
|
template<typename T>
|
||||||
auto state(state_.lock());
|
void computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges)
|
||||||
while (state->pending)
|
{
|
||||||
state.wait(done);
|
asio::io_context ctx;
|
||||||
if (state->exc)
|
std::exception_ptr ex = nullptr;
|
||||||
std::rethrow_exception(state->exc);
|
computeClosure(
|
||||||
}
|
std::move(startElts),
|
||||||
|
res,
|
||||||
|
std::move(getEdges),
|
||||||
|
asio::bind_executor(ctx.get_executor(), [&](std::exception_ptr ex2) { ex = ex2; }));
|
||||||
|
ctx.run();
|
||||||
|
if (ex)
|
||||||
|
std::rethrow_exception(ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -16,6 +16,22 @@ struct CompressionSink : BufferedSink, FinishSink
|
|||||||
using FinishSink::finish;
|
using FinishSink::finish;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
enum class CompressionAlgo {
|
||||||
|
none,
|
||||||
|
brotli,
|
||||||
|
bzip2,
|
||||||
|
compress,
|
||||||
|
grzip,
|
||||||
|
gzip,
|
||||||
|
lrzip,
|
||||||
|
lz4,
|
||||||
|
lzip,
|
||||||
|
lzma,
|
||||||
|
lzop,
|
||||||
|
xz,
|
||||||
|
zstd,
|
||||||
|
};
|
||||||
|
|
||||||
std::string decompress(const std::string & method, std::string_view in);
|
std::string decompress(const std::string & method, std::string_view in);
|
||||||
|
|
||||||
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
||||||
@@ -25,6 +41,9 @@ std::string compress(const std::string & method, std::string_view in, const bool
|
|||||||
ref<CompressionSink>
|
ref<CompressionSink>
|
||||||
makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
|
makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
|
||||||
|
|
||||||
|
ref<CompressionSink>
|
||||||
|
makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel = false, int level = -1);
|
||||||
|
|
||||||
MakeError(UnknownCompressionMethod, Error);
|
MakeError(UnknownCompressionMethod, Error);
|
||||||
|
|
||||||
MakeError(CompressionError, Error);
|
MakeError(CompressionError, Error);
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
///@file
|
///@file
|
||||||
|
|
||||||
#include "nix/util/canon-path.hh"
|
#include "nix/util/canon-path.hh"
|
||||||
#include "nix/util/types.hh"
|
|
||||||
#include "nix/util/error.hh"
|
#include "nix/util/error.hh"
|
||||||
|
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
@@ -236,18 +235,6 @@ std::wstring handleToFileName(Descriptor handle);
|
|||||||
#ifndef _WIN32
|
#ifndef _WIN32
|
||||||
namespace unix {
|
namespace unix {
|
||||||
|
|
||||||
struct SymlinkNotAllowed : public Error
|
|
||||||
{
|
|
||||||
CanonPath path;
|
|
||||||
|
|
||||||
SymlinkNotAllowed(CanonPath path)
|
|
||||||
/* Can't provide better error message, since the parent directory is only known to the caller. */
|
|
||||||
: Error("relative path '%s' points to a symlink, which is not allowed", path.rel())
|
|
||||||
, path(std::move(path))
|
|
||||||
{
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Safe(r) function to open \param path file relative to \param dirFd, while
|
* Safe(r) function to open \param path file relative to \param dirFd, while
|
||||||
* disallowing escaping from a directory and resolving any symlinks in the
|
* disallowing escaping from a directory and resolving any symlinks in the
|
||||||
@@ -274,4 +261,14 @@ Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & p
|
|||||||
|
|
||||||
MakeError(EndOfFile, Error);
|
MakeError(EndOfFile, Error);
|
||||||
|
|
||||||
|
#ifdef _WIN32
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Windows specific replacement for POSIX `lseek` that operates on a `HANDLE` and not
|
||||||
|
* a file descriptor.
|
||||||
|
*/
|
||||||
|
off_t lseek(Descriptor fd, off_t offset, int whence);
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
} // namespace nix
|
} // namespace nix
|
||||||
@@ -40,6 +40,11 @@ struct UnixPathTrait
|
|||||||
{
|
{
|
||||||
return path.rfind('/', from);
|
return path.rfind('/', from);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static size_t rootNameLen(StringView)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -83,6 +88,18 @@ struct WindowsPathTrait
|
|||||||
size_t p2 = path.rfind(preferredSep, from);
|
size_t p2 = path.rfind(preferredSep, from);
|
||||||
return p1 == String::npos ? p2 : p2 == String::npos ? p1 : std::max(p1, p2);
|
return p1 == String::npos ? p2 : p2 == String::npos ? p1 : std::max(p1, p2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static size_t rootNameLen(StringView path)
|
||||||
|
{
|
||||||
|
if (path.size() >= 2 && path[1] == ':') {
|
||||||
|
char driveLetter = path[0];
|
||||||
|
if ((driveLetter >= 'A' && driveLetter <= 'Z') || (driveLetter >= 'a' && driveLetter <= 'z'))
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
/* TODO: This needs to also handle UNC paths.
|
||||||
|
* https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats#unc-paths */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
template<typename CharT>
|
template<typename CharT>
|
||||||
@@ -116,6 +133,11 @@ typename PathDict::String canonPathInner(typename PathDict::StringView remaining
|
|||||||
typename PathDict::String result;
|
typename PathDict::String result;
|
||||||
result.reserve(256);
|
result.reserve(256);
|
||||||
|
|
||||||
|
if (auto rootNameLength = PathDict::rootNameLen(remaining)) {
|
||||||
|
result += remaining.substr(0, rootNameLength); /* Copy drive letter verbatim. */
|
||||||
|
remaining.remove_prefix(rootNameLength);
|
||||||
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
|
||||||
/* Skip slashes. */
|
/* Skip slashes. */
|
||||||
|
|||||||
@@ -362,6 +362,8 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Return `TMPDIR`, or the default temporary directory if unset or empty.
|
* Return `TMPDIR`, or the default temporary directory if unset or empty.
|
||||||
|
* Uses GetTempPathW on windows which respects TMP, TEMP, USERPROFILE env variables.
|
||||||
|
* Does not resolve symlinks and the returned path might not be directory or exist at all.
|
||||||
*/
|
*/
|
||||||
std::filesystem::path defaultTempDir();
|
std::filesystem::path defaultTempDir();
|
||||||
|
|
||||||
@@ -480,8 +482,23 @@ class AutoUnmount
|
|||||||
Path path;
|
Path path;
|
||||||
bool del;
|
bool del;
|
||||||
public:
|
public:
|
||||||
AutoUnmount(Path &);
|
|
||||||
AutoUnmount();
|
AutoUnmount();
|
||||||
|
AutoUnmount(Path &);
|
||||||
|
AutoUnmount(const AutoUnmount &) = delete;
|
||||||
|
|
||||||
|
AutoUnmount(AutoUnmount && other) noexcept
|
||||||
|
: path(std::move(other.path))
|
||||||
|
, del(std::exchange(other.del, false))
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
AutoUnmount & operator=(AutoUnmount && other) noexcept
|
||||||
|
{
|
||||||
|
path = std::move(other.path);
|
||||||
|
del = std::exchange(other.del, false);
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
~AutoUnmount();
|
~AutoUnmount();
|
||||||
void cancel();
|
void cancel();
|
||||||
};
|
};
|
||||||
|
|||||||
Loaded 100 of 130 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user