Compare commits

..

1 Commits

Author SHA1 Message Date
John Ericson
da9ade20e3 Add warning for non-JSON-object exportReferencesGraph 2025-12-09 13:02:46 -05:00
73 changed files with 672 additions and 1797 deletions

View File

@@ -24,8 +24,8 @@ inputs:
description: "Github token"
required: true
use_cache:
description: "Whether to setup github actions cache (not implemented currently)"
default: false
description: "Whether to setup magic-nix-cache"
default: true
required: false
runs:
using: "composite"
@@ -122,3 +122,10 @@ runs:
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
extra-conf: ${{ inputs.extra_nix_config }}
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
if: ${{ inputs.use_cache == 'true' }}
with:
diagnostic-endpoint: ''
use-flakehub: false
use-gha-cache: true
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main

View File

@@ -26,7 +26,7 @@ jobs:
# required to find all branches
fetch-depth: 0
- name: Create backport PRs
uses: korthout/backport-action@c656f5d5851037b2b38fb5db2691a03fa229e3b2 # v4.0.1
uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
id: backport
with:
# Config README: https://github.com/korthout/backport-action#backport-action

View File

@@ -164,7 +164,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- name: Download installer tarball
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
uses: actions/download-artifact@v6
with:
name: installer-${{matrix.os}}
path: out
@@ -197,20 +197,79 @@ jobs:
- run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
- run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
# Steps to test CI automation in your own fork.
# 1. Sign-up for https://hub.docker.com/
# 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
# 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
check_secrets:
permissions:
contents: none
name: Check presence of secrets
runs-on: ubuntu-24.04
outputs:
docker: ${{ steps.secret.outputs.docker }}
steps:
- name: Check for DockerHub secrets
id: secret
env:
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
run: |
echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
docker_push_image:
name: Push docker image to DockerHub and GHCR
needs: [flake_regressions, installer_test]
if: github.event_name == 'push' && github.ref_name == 'master'
uses: ./.github/workflows/docker-push.yml
with:
ref: ${{ github.sha }}
is_master: true
needs: [tests, check_secrets]
permissions:
contents: read
packages: write
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
if: >-
needs.check_secrets.outputs.docker == 'true' &&
github.event_name == 'push' &&
github.ref_name == 'master'
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: ./.github/actions/install-nix-action
with:
dogfood: false
extra_nix_config: |
experimental-features = flakes nix-command
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
- run: nix build .#dockerImage -L
- run: docker load -i ./result/image.tar.gz
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
#
# Push to Docker Hub first
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
# Push to GitHub Container Registry as well
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push image
run: |
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
# Change all uppercase to lowercase
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
docker tag nix:$NIX_VERSION $IMAGE_ID:latest
docker push $IMAGE_ID:$NIX_VERSION
docker push $IMAGE_ID:latest
# deprecated 2024-02-24
docker tag nix:$NIX_VERSION $IMAGE_ID:master
docker push $IMAGE_ID:master
flake_regressions:
needs: tests
@@ -228,21 +287,13 @@ jobs:
with:
repository: NixOS/flake-regressions-data
path: flake-regressions/tests
- name: Download installer tarball
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- uses: ./.github/actions/install-nix-action
with:
name: installer-linux
path: out
- name: Looking up the installer tarball URL
id: installer-tarball-url
run: |
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }}
- name: Run flake regressions tests
run: MAX_FLAKES=25 flake-regressions/eval-all.sh
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
extra_nix_config:
experimental-features = nix-command flakes
github_token: ${{ secrets.GITHUB_TOKEN }}
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
profile_build:
needs: tests

View File

@@ -1,101 +0,0 @@
name: "Push Docker Image"
on:
workflow_call:
inputs:
ref:
description: "Git ref to build the docker image from"
required: true
type: string
is_master:
description: "Whether run from master branch"
required: true
type: boolean
secrets:
DOCKERHUB_USERNAME:
required: true
DOCKERHUB_TOKEN:
required: true
permissions: {}
jobs:
# Steps to test CI automation in your own fork.
# 1. Sign-up for https://hub.docker.com/
# 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
# 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
check_secrets:
permissions:
contents: none
name: Check presence of secrets
runs-on: ubuntu-24.04
outputs:
docker: ${{ steps.secret.outputs.docker }}
steps:
- name: Check for DockerHub secrets
id: secret
env:
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
run: |
echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
push:
name: Push docker image to DockerHub and GHCR
needs: [check_secrets]
permissions:
contents: read
packages: write
if: needs.check_secrets.outputs.docker == 'true'
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
fetch-depth: 0
ref: ${{ inputs.ref }}
- uses: ./.github/actions/install-nix-action
with:
dogfood: false
extra_nix_config: |
experimental-features = flakes nix-command
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
- run: nix build .#dockerImage -L
- run: docker load -i ./result/image.tar.gz
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
#
# Push to Docker Hub first
- name: Login to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Push to Docker Hub
env:
IS_MASTER: ${{ inputs.is_master }}
DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_USERNAME }}/nix
run: |
docker tag nix:$NIX_VERSION $DOCKERHUB_REPO:$NIX_VERSION
docker push $DOCKERHUB_REPO:$NIX_VERSION
if [ "$IS_MASTER" = "true" ]; then
docker tag nix:$NIX_VERSION $DOCKERHUB_REPO:master
docker push $DOCKERHUB_REPO:master
fi
# Push to GitHub Container Registry as well
- name: Login to GitHub Container Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push to GHCR
env:
IS_MASTER: ${{ inputs.is_master }}
run: |
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
docker push $IMAGE_ID:$NIX_VERSION
if [ "$IS_MASTER" = "true" ]; then
docker tag nix:$NIX_VERSION $IMAGE_ID:master
docker push $IMAGE_ID:master
fi

View File

@@ -1,69 +0,0 @@
name: Upload Release
on:
workflow_dispatch:
inputs:
eval_id:
description: "Hydra evaluation ID"
required: true
type: number
is_latest:
description: "Mark as latest release"
required: false
type: boolean
default: false
permissions:
contents: read
id-token: write
packages: write
jobs:
release:
runs-on: ubuntu-24.04
environment: releases
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: ./.github/actions/install-nix-action
with:
dogfood: false # Use stable version
use_cache: false # Don't want any cache injection shenanigans
extra_nix_config: |
experimental-features = nix-command flakes
- name: Set NIX_PATH from flake input
run: |
NIXPKGS_PATH=$(nix build --inputs-from .# nixpkgs#path --print-out-paths --no-link)
# Shebangs with perl have issues. Pin nixpkgs this way. nix shell should maybe
# get the same uberhack that nix-shell has to support it.
echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
with:
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
role-session-name: nix-release-oidc-${{ github.run_id }}
aws-region: eu-west-1
- name: Login to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Upload release
run: |
./maintainers/upload-release.pl \
${{ inputs.eval_id }} \
--skip-git
env:
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
- name: Push to GHCR
run: |
DOCKER_OWNER="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[A-Z]' '[a-z]')/nix"
./maintainers/upload-release.pl \
${{ inputs.eval_id }} \
--skip-git \
--skip-s3 \
--docker-owner "$DOCKER_OWNER"
env:
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}

View File

@@ -1 +1 @@
2.33.2
2.33.0

View File

@@ -0,0 +1,9 @@
---
synopsis: Channel URLs migrated to channels.nixos.org subdomain
prs: [14518]
issues: [14517]
---
Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix.
The subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.

View File

@@ -0,0 +1,88 @@
---
synopsis: "JSON format changes for store path info and derivations"
prs: []
issues: []
---
JSON formats for store path info and derivations have been updated with new versions and structured fields.
## Store Path Info JSON
`nix path-info --json` now requires a `--json-format` flag to specify the output format version.
Using `--json` without `--json-format` is deprecated and will become an error in a future release.
For now, it defaults to version 1 with a warning, for a smoother migration.
### Version 1 (`--json-format 1`)
This is the legacy format, preserved for backwards compatibility:
- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
- Now includes `"storeDir"` field at the top level
### Version 2 (`--json-format 2`)
The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
- **Nested structure with top-level metadata**:
The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
```json
{
"version": 2,
"storeDir": "/nix/store",
"info": { ... }
}
```
The map from store bath base names to store object info is nested under the `info` field.
- **Store path base names instead of full paths**:
Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
Combined with `storeDir`, the full path can be reconstructed.
- **Structured `ca` field**:
Content address is now a structured JSON object instead of a string:
- Old: `"ca": "fixed:r:sha256:1abc..."`
- New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
- Still `null` values for input-addressed store objects
The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
Nix currently only produces, and doesn't consume this format.
Additionally the following field is added to both formats.
(The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.)
- **`version` field**:
All store path info JSON now includes `"version": <1|2>`.
- **`storeDir` field**:
Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
## Derivation JSON (Version 4)
The derivation JSON format has been updated from version 3 to version 4:
- **Restructured inputs**:
Inputs are now nested under an `inputs` object:
- Old: `"inputSrcs": [...], "inputDrvs": {...}`
- New: `"inputs": {"srcs": [...], "drvs": {...}}`
- **Consistent content addresses**:
Fixed content-addressed outputs now use structured JSON format.
This is the same format as `ca` in store path info (after the new version).
Version 3 and earlier formats are *not* accepted when reading.
**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.

View File

@@ -0,0 +1,12 @@
---
synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
prs: [14614]
issues: [11728]
---
The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

View File

@@ -0,0 +1,8 @@
---
synopsis: Interrupting REPL commands works more than once
issues: [13481]
---
Previously, this only worked once per REPL session; further attempts would be ignored.
This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).

View File

@@ -0,0 +1,40 @@
---
synopsis: "Improved S3 binary cache support via HTTP"
prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421]
issues: [13084, 12671, 11748, 12403]
---
S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native
AWS SigV4 authentication instead of the AWS C++ SDK, providing significant
improvements:
- **Reduced memory usage**: Eliminates memory buffering issues that caused
segfaults with large files
- **Fixed upload reliability**: Resolves AWS SDK chunking errors
(`InvalidChunkSizeError`)
- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full
`aws-cpp-sdk`, reducing build complexity
The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
management.
All existing S3 URL formats and parameters remain supported, however the store
settings for configuring multipart uploads have changed:
- **`multipart-upload`** (default: `false`): Enable multipart uploads for large
files. When enabled, files exceeding the multipart threshold will be uploaded
in multiple parts.
- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using
multipart uploads. Files smaller than this will use regular PUT requests.
Only takes effect when `multipart-upload` is enabled.
- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart
uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes
reduce the number of requests but use more memory.
- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
Note that this change also means Nix now supports S3 binary cache stores even
if built without `aws-crt-cpp`, but only for public buckets which do not
require authentication.

View File

@@ -0,0 +1,14 @@
---
synopsis: "S3 URLs now support object versioning via versionId parameter"
prs: [14274]
issues: [13955]
---
S3 URLs now support a `versionId` query parameter to fetch specific versions
of objects from S3 buckets with versioning enabled. This allows pinning to
exact object versions for reproducibility and protection against unexpected
changes:
```
s3://bucket/key?region=us-east-1&versionId=abc123def456
```

View File

@@ -0,0 +1,21 @@
---
synopsis: "S3 binary cache stores now support storage class configuration"
prs: [14464]
issues: [7015]
---
S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
Example usage:
```bash
# Use Glacier storage for long-term archival
nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
# Use Intelligent Tiering for automatic cost optimization
nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
```
The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.

View File

@@ -151,7 +151,6 @@
- [Contributing](development/contributing.md)
- [Releases](release-notes/index.md)
{{#include ./SUMMARY-rl-next.md}}
- [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md)
- [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md)
- [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md)
- [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)

View File

@@ -1,281 +0,0 @@
# Release 2.33.0 (2025-12-09)
## New features
- New command `nix registry resolve` [#14595](https://github.com/NixOS/nix/pull/14595)
This command looks up a flake registry input name and returns the flakeref it resolves to.
For example, looking up Nixpkgs:
```
$ nix registry resolve nixpkgs
github:NixOS/nixpkgs/nixpkgs-unstable
```
Upstreamed from [Determinate Nix 3.14.0](https://github.com/DeterminateSystems/nix-src/pull/273).
- `nix flake clone` supports all input types [#14581](https://github.com/NixOS/nix/pull/14581)
`nix flake clone` now supports arbitrary input types. In particular, this allows you to clone tarball flakes, such as flakes on FlakeHub.
Upstreamed from [Determinate Nix 3.12.0](https://github.com/DeterminateSystems/nix-src/pull/229).
## Performance improvements
- Git fetcher computes `revCount`s using multiple threads [#14462](https://github.com/NixOS/nix/pull/14462)
When using Git repositories with a long history, calculating the `revCount` attribute can take a long time. Nix now computes `revCount` using multiple threads, making it much faster (e.g. 9.1s to 3.7s for Nixpkgs).
Note that if you don't need `revCount`, you can disable it altogether by setting the flake input attribute `shallow = true`.
Upstreamed from [Determinate Nix 3.12.2](https://github.com/DeterminateSystems/nix-src/pull/245).
- `builtins.stringLength` now runs in constant time [#14442](https://github.com/NixOS/nix/pull/14442)
The internal representation of strings has been replaced with a size-prefixed Pascal style string. Previously Nix stored strings as a NUL-terminated array of bytes, necessitating a linear scan to calculate the length.
- Uploads to `http://` and `https://` binary cache stores now run in constant memory [#14390](https://github.com/NixOS/nix/pull/14390)
Nix used to buffer the whole compressed NAR contents in memory. It now reads it in a streaming fashion.
- Channel URLs migrated to channels.nixos.org subdomain [#14517](https://github.com/NixOS/nix/issues/14517) [#14518](https://github.com/NixOS/nix/pull/14518)
Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix. This subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.
- Fix `download buffer is full; consider increasing the 'download-buffer-size' setting` warning [#11728](https://github.com/NixOS/nix/issues/11728) [#14614](https://github.com/NixOS/nix/pull/14614)
The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
- Significantly improve tarball unpacking performance [#14689](https://github.com/NixOS/nix/pull/14689) [#14696](https://github.com/NixOS/nix/pull/14696) [#10683](https://github.com/NixOS/nix/issues/10683) [#11098](https://github.com/NixOS/nix/issues/11098)
Nix uses a content-addressed cache backed by libgit2 for deduplicating files fetched via `fetchTarball` and `github`, `tarball` flake inputs. Its usage has been significantly optimised to reduce the amount of I/O operations that are performed. For a typical nixpkgs source tarball this results in 200 times fewer system calls on Linux. In combination with libcurl pausing this alleviates performance regressions stemming from the tarball cache.
- Already valid derivations are no longer copied to the store [#14219](https://github.com/NixOS/nix/pull/14219)
This results in a modest speedup when using the Nix daemon.
- `nix nar ls` and `nix nar cat` are significantly faster and no longer buffer the whole NAR in memory [#14273](https://github.com/NixOS/nix/pull/14273) [#14732](https://github.com/NixOS/nix/pull/14732)
## S3 improvements
- Improved S3 binary cache support via HTTP [#11748](https://github.com/NixOS/nix/issues/11748) [#12403](https://github.com/NixOS/nix/issues/12403) [#12671](https://github.com/NixOS/nix/issues/12671) [#13084](https://github.com/NixOS/nix/issues/13084) [#13752](https://github.com/NixOS/nix/pull/13752) [#13823](https://github.com/NixOS/nix/pull/13823) [#14026](https://github.com/NixOS/nix/pull/14026) [#14120](https://github.com/NixOS/nix/pull/14120) [#14131](https://github.com/NixOS/nix/pull/14131) [#14135](https://github.com/NixOS/nix/pull/14135) [#14144](https://github.com/NixOS/nix/pull/14144) [#14170](https://github.com/NixOS/nix/pull/14170) [#14190](https://github.com/NixOS/nix/pull/14190) [#14198](https://github.com/NixOS/nix/pull/14198) [#14206](https://github.com/NixOS/nix/pull/14206) [#14209](https://github.com/NixOS/nix/pull/14209) [#14222](https://github.com/NixOS/nix/pull/14222) [#14223](https://github.com/NixOS/nix/pull/14223) [#14330](https://github.com/NixOS/nix/pull/14330) [#14333](https://github.com/NixOS/nix/pull/14333) [#14335](https://github.com/NixOS/nix/pull/14335) [#14336](https://github.com/NixOS/nix/pull/14336) [#14337](https://github.com/NixOS/nix/pull/14337) [#14350](https://github.com/NixOS/nix/pull/14350) [#14356](https://github.com/NixOS/nix/pull/14356) [#14357](https://github.com/NixOS/nix/pull/14357) [#14374](https://github.com/NixOS/nix/pull/14374) [#14375](https://github.com/NixOS/nix/pull/14375) [#14376](https://github.com/NixOS/nix/pull/14376) [#14377](https://github.com/NixOS/nix/pull/14377) [#14391](https://github.com/NixOS/nix/pull/14391) [#14393](https://github.com/NixOS/nix/pull/14393) [#14420](https://github.com/NixOS/nix/pull/14420) [#14421](https://github.com/NixOS/nix/pull/14421)
S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native AWS SigV4 authentication instead of the AWS C++ SDK, providing significant improvements:
- **Reduced memory usage**: Eliminates memory buffering issues that caused segfaults with large files
- **Fixed upload reliability**: Resolves AWS SDK chunking errors (`InvalidChunkSizeError`)
- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full `aws-cpp-sdk`, reducing build complexity
The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential management.
All existing S3 URL formats and parameters remain supported, however the store settings for configuring multipart uploads have changed:
- **`multipart-upload`** (default: `false`): Enable multipart uploads for large files. When enabled, files exceeding the multipart threshold will be uploaded in multiple parts.
- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using multipart uploads. Files smaller than this will use regular PUT requests. Only takes effect when `multipart-upload` is enabled.
- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the number of requests but use more memory.
- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
Note that this change also means Nix now supports S3 binary cache stores even if built without `aws-crt-cpp`, but only for public buckets which do not require authentication.
- S3 URLs now support object versioning via `versionId` parameter [#13955](https://github.com/NixOS/nix/issues/13955) [#14274](https://github.com/NixOS/nix/pull/14274)
S3 URLs now support a `versionId` query parameter to fetch specific versions
of objects from S3 buckets with versioning enabled. This allows pinning to
exact object versions for reproducibility and protection against unexpected
changes:
```
s3://bucket/key?region=us-east-1&versionId=abc123def456
```
- S3 binary cache stores now support storage class configuration [#7015](https://github.com/NixOS/nix/issues/7015) [#14464](https://github.com/NixOS/nix/pull/14464)
S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
Example usage:
```bash
# Use Glacier storage for long-term archival
nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
# Use Intelligent Tiering for automatic cost optimization
nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
```
The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.
## Store path info JSON format changes
The JSON format emitted by `nix path-info --json` has been updated to a new version with improved structure.
To maintain compatibility, `nix path-info --json` now requires a `--json-format` flag to specify the output format version.
Using `--json` without `--json-format` is deprecated and will become an error in a future release.
For now, it defaults to version 1 with a warning, for a smoother migration.
### Version 1 (`--json-format 1`)
This is the legacy format, preserved for backwards compatibility:
- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
- Now includes `"storeDir"` field at the top level
### Version 2 (`--json-format 2`)
The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
- **Nested structure with top-level metadata**:
The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
```json
{
"version": 2,
"storeDir": "/nix/store",
"info": { ... }
}
```
The map from store path base names to store object info is nested under the `info` field.
- **Store path base names instead of full paths**:
Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
Combined with `storeDir`, the full path can be reconstructed.
- **Structured `ca` field**:
Content address is now a structured JSON object instead of a string:
- Old: `"ca": "fixed:r:sha256:1abc..."`
- New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
- Still `null` values for input-addressed store objects
The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
Additionally the following fields are added to both formats:
- **`version` field**:
All store path info JSON now includes `"version": <1|2>`. The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.
- **`storeDir` field**:
Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
## Derivation JSON format changes
The derivation JSON format has been updated from version 3 to version 4:
- **Nested structure with top-level metadata**:
The output of `nix derivation show` is now wrapped in an object with `version` and `derivations` fields:
```json
{
"version": 4,
"derivations": { ... }
}
```
The map from derivation paths to derivation info is nested under the `derivations` field.
This matches the structure used for `nix path-info --json --json-format 2`, and likewise brings this command into compliance with the JSON guidelines.
- **Restructured inputs**:
Inputs are now nested under an `inputs` object:
- Old: `"inputSrcs": [...], "inputDrvs": {...}`
- New: `"inputs": {"srcs": [...], "drvs": {...}}`
- **Consistent content addresses**:
Fixed content-addressed outputs now use structured JSON format.
This is the same format as `ca` in store path info (after the new version).
Version 3 and earlier formats are *not* accepted when reading.
**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.
## Miscellaneous changes
- Git fetcher: Restore progress indication [#14487](https://github.com/NixOS/nix/pull/14487)
Nix used to feel "stuck" while it was cloning large repositories. Nix now shows Git's native progress indicator while fetching.
Upstreamed from [Determinate Nix 3.13.0](https://github.com/DeterminateSystems/nix-src/pull/250).
- Interrupting REPL commands works more than once [#13481](https://github.com/NixOS/nix/issues/13481)
Previously, this only worked once per REPL session; further attempts would be ignored.
This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).
- NAR unpacking code has been rewritten to make use of dirfd-based `openat` and `openat2` system calls when available [#14597](https://github.com/NixOS/nix/pull/14597)
- Dynamic size unit rendering [#14423](https://github.com/NixOS/nix/pull/14423) [#14364](https://github.com/NixOS/nix/pull/14364)
Various commands and the progress bar now use dynamically determined size units instead
of always using `MiB`. For example, the progress bar now reports download status like:
```
[1/196/197 copied (773.7 MiB/2.1 GiB), 172.4/421.5 MiB DL]
```
Instead of:
```
[1/196/197 copied (773.7/2147.3 MiB), 172.4/421.5 MiB DL]
```
## Contributors
This release was made possible by the following 33 contributors:
- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and)
- jonhermansen [**(@jonhermansen)**](https://github.com/jonhermansen)
- Arnout Engelen [**(@raboof)**](https://github.com/raboof)
- Jean-François Roche [**(@jfroche)**](https://github.com/jfroche)
- tomberek [**(@tomberek)**](https://github.com/tomberek)
- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra)
- Marcel [**(@MarcelCoding)**](https://github.com/MarcelCoding)
- David McFarland [**(@corngood)**](https://github.com/corngood)
- Soumyadip Sarkar [**(@neuralsorcerer)**](https://github.com/neuralsorcerer)
- Cole Helbling [**(@cole-h)**](https://github.com/cole-h)
- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314)
- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy)
- Alex Auvolat [**(@Alexis211)**](https://github.com/Alexis211)
- edef [**(@edef1c)**](https://github.com/edef1c)
- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium)
- Vinayak Goyal [**(@vinayakankugoyal)**](https://github.com/vinayakankugoyal)
- Graham Dennis [**(@GrahamDennis)**](https://github.com/GrahamDennis)
- Aspen Smith [**(@glittershark)**](https://github.com/glittershark)
- Jens Petersen [**(@juhp)**](https://github.com/juhp)
- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault)
- Peter Bynum [**(@pkpbynum)**](https://github.com/pkpbynum)
- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92)
- Alex Decious [**(@adeci)**](https://github.com/adeci)
- Matthieu Coudron [**(@teto)**](https://github.com/teto)
- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar)
- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii)
- Seth Flynn [**(@getchoo)**](https://github.com/getchoo)
- Robert Hensing [**(@roberth)**](https://github.com/roberth)
- Vladimir Panteleev [**(@CyberShadow)**](https://github.com/CyberShadow)
- bryango [**(@bryango)**](https://github.com/bryango)
- Henry [**(@cootshk)**](https://github.com/cootshk)
- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)

View File

@@ -32,7 +32,7 @@
let
inherit (nixpkgs) lib;
officialRelease = true;
officialRelease = false;
linux32BitSystems = [ "i686-linux" ];
linux64BitSystems = [

View File

@@ -224,25 +224,5 @@
"42688647+netadr@users.noreply.github.com": "netadr",
"matej.urbas@gmail.com": "urbas",
"ethanalexevans@gmail.com": "ethanavatar",
"greg.marti@gmail.com": "gmarti",
"arnout@bzzt.net": "raboof",
"vinayakankugoyal@gmail.com": "vinayakankugoyal",
"Radvendii@users.noreply.github.com": "Radvendii",
"jon@jh86.org": "jonhermansen",
"edef@edef.eu": "edef1c",
"pkpbynum@gmail.com": "pkpbynum",
"886074+teto@users.noreply.github.com": "teto",
"alex@adnab.me": "Alexis211",
"root@gws.fyi": "glittershark",
"me@m4rc3l.de": "MarcelCoding",
"taeer.bar-yam@bevuta.com": "Radvendii",
"martin.joerg@gmail.com": "mjoerg",
"git@cy.md": "CyberShadow",
"cootshk@duck.com": "cootshk",
"adam@dinwoodie.org": "me-and",
"domen@cachix.org": "domenkozar",
"alex.decious@gmail.com": "adeci",
"soumya.papanvk18@gmail.com": "neuralsorcerer",
"gdennis@anduril.com": null,
"graham.dennis@gmail.com": "GrahamDennis"
"greg.marti@gmail.com": "gmarti"
}

View File

@@ -196,21 +196,5 @@
"gmarti": "Gr\u00e9gory Marti",
"lovesegfault": "Bernardo Meurer",
"EphraimSiegfried": "Ephraim Siegfried",
"hgl": "Glen Huang",
"mjoerg": "Martin Joerg",
"Alexis211": "Alex Auvolat",
"domenkozar": "Domen Ko\u017ear",
"edef1c": "edef",
"cootshk": "Henry",
"raboof": "Arnout Engelen",
"pkpbynum": "Peter Bynum",
"glittershark": "Aspen Smith",
"MarcelCoding": "Marcel",
"teto": "Matthieu Coudron",
"jonhermansen": null,
"neuralsorcerer": "Soumyadip Sarkar",
"adeci": "Alex Decious",
"vinayakankugoyal": "Vinayak Goyal",
"me-and": "Adam Dinwoodie",
"GrahamDennis": "Graham Dennis"
"hgl": "Glen Huang"
}

View File

@@ -1,110 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGPtMiwBEAC0sFZW2QW/OaDjKm5zGRpDvHXDsMIUtlHfoi5ce8pocC63W05o
FSXbUZjZ1VfYO8lT8DFANCzTkiXYaZx0cPRG2pVY4AOQZDNFt5XrAyvw496XCAIM
DTYGFLjCqgjPt9RUFEy4MyHPJTEpB0x3rXgT4ILNu9vsj9Q0vttps7SpbZ3Ldq5H
o/BBbLW77q/vNjpYzCbBIXF7ycUGpnNv9Go/WuiDnrBMcyxh+8kjjIHB5cxZSnjJ
DUv681+m83v+gLZQGX/jexQrrf5JpS0X9qEnhGLrNUDhtyv5ud3Je4EfamkjLVVC
RlNLofgflOCsl/tP80i+K7S1QdKhUALxuJ6H0prYUflGBDxDyC8XYuJ62TT0OUpa
vJvgwVlCq8/jq+ykYQXlbuBVOzi5wAuI4l3+HqreSQYPSiwe+6N590Zbafdv1fvN
WFtZKCTGMqfyaaAnppioH9/+NWkI2AQxaYVasYM/JEYvY9pJgA7alh51jHW4JglP
ErypKfBKPKJID0QENqYoa3bDDCihuNWhgQf9dxzPlj2ckd35Zb6w4DfuSmtjaa9D
o0jZVY1JbFuxBqP09+saVPrxLHgmPxjcdzPGQQtAqdO2vyJXNEGLFMoVEZPNaLo3
QmcIJnT7oSck+4vGfOYtWUHXQynu/Tnwsv2XkA/uyw8HNe+RRMqv/apnzQARAQAB
tCdTZXJnZWkgWmltbWVybWFuIDxzZXJnZWlAemltbWVybWFuLmZvbz6JAlEEEwEK
ADsCGwEFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQQVim9TDqIC5fZRYRMU+upj
RI4d+QUCaUbsaAIZAQAKCRAU+upjRI4d+VdDD/492HRaJ/8V7R7VUzkafmb2Hb28
SLf7oiB8Uq9I7SukiDEaIT1fUhquYWQ9KWpPRNR1TX6ApXnIeuJRMGFoDVIRnmnr
cKnYYXfqqc81VxIyKvaumB7KWbS7G4Nbor8AH1ouOOOMMS50OTJOWQA4A26inIuG
n+7L8MeS5aT+3uNKDoTKsidC47vnaxNMcke1taPfbfo7vn69PsRCM/g9/7TQYU8b
6xp+pM9Ao9nJneRk2YCpsGYRrWTpaik0DFKnfpPKJM/yunhtGLF2IYAp3l1mvHPK
nnzo92zjpQuZwazEIK+23V1vRT4IjM2BewbJPAzf2/UuxEjjgNQm0tOtH2JhFNeB
VM0BVrGxWrrwrsmv6lWghTtBc6zRWyHrj/rpjtVQNmeKYrHWJeXwVz1rqgGPmB2N
k0MZD1UjHHhEs1Cntn7yLmxTPztRJCtR+euRu81Uo2NAvrMJ4xsDjaM0LeLTnzjV
9AsPjD188dOFyz7VExZum4+XaaEJ41FIPLEqU3U0GAa6stEy0ylSlIN4x9aiXXVW
xfzHHchS5jK6QAjuZxN8t01GRactNylINRf7uoTECFZTtXNqfeuk0HQxBH0LuKVE
0PxJbcNI4mVWw1KgTJ8PUVC1IXP3sEpqPdJOYiRXgnpcS26fWOBu0aQ4mhxiaJhr
/zBfrkLEqp20TdNDZLkCDQRj7TM1ARAAt73xO24curnHTTgXkkVMMRzcMLx3Mb1a
2FuddxC5hzTpEpw01L91UBrXVJEg9K2KAwP5CtCLgPCqXr47Tm7krvHxWwBksgY/
6aHRsoPQfCFUZHc0aiO+C4NCzR+aEeGKn66Oc1Hq9oUTpDgiBWhsuEPiyA1OSGF0
4L0jeTCqfm68kWp4PIK9yuugkdDsoyj6TonuMsb3V5ctHLqop9KH+eHSkUTPo+Lk
+bxaeAOJ1UfbohgbRbrYKAfsaghhOMDH3R1w2pvtUJz+sDbuQsiPFTqbxsXDTFws
H4N/AQCYnnvOhqEek2sOEZ19bJXt5UrAr10mX4PGmAkWqE1JWBxpOKG3BXSGOTu1
3dFhQfPMK+PmvUrs0kcWQr53K/aRUdKKhIfTcMfkqYTGPK5HclHph24WjXj3QFFA
SjksQTdm6486ZmLZK4CTbAFOPfTF/aWg8gu9v4ihdq6lqHNNXxv2xBAChcd59H7p
D5zy9z8SpwWR9V5JDmlF6HWIIau1c6lSsQq1xHvYM8EuPe03vJvor+2u/cn5zYF1
5ZxAuPI2i5vtavg1s8ZGAAogJ9dVcP36LdJfL9quXWvmovkd//qHIepBB+l/zQio
ZRDZlIcfV3Xycaqsb5OqHGARHE0097koipMt5y/iXlqG4Ruue6Idb8bW96EKpaWj
kKy/iNfQfQMAEQEAAYkEcgQYAQoAJgIbAhYhBBWKb1MOogLl9lFhExT66mNEjh35
BQJpRuz3BQkJHCDCAkDBdCAEGQEKAB0WIQRK3hK0WyJ4BicGpcmpsLVXymMjJQUC
Y+0zNQAKCRCpsLVXymMjJbdSD/9+f1FOOeGDAJI6Duo5fsWnf4xJJdtQtDbz6d2A
SeDapxeJ3zWfKBD0wu5sISEa0uiWsYSmLtsa2SqVAKHlEaMGRR+tkBMPQ+rvgI4c
62YjGTgm+IPd+NFIn+ixFU1hpinTh+KhUEoeOwWCvKs9nZfSG9vkienfiG0bBxo2
zrvBzXA50x5hbUL+ghKu/AVfN9qZDwh30O4KZTwk4g4cM9SeaQa4YvHYIS3IEhDZ
hGybwrrqV9cs92ln4IJw9WCy9QReBNrdeFgC4+3ziUp1QsG3RvqrtuMttwBVC1Z3
bj5QjLLOREhhodfvk98t9yVkragObb4rGrLo1mWuF0c4mJGvXwnrqhCMvzv4M+0T
Zdrmw6YpGkGOaOPghVuwoTtqSAkl+zFWIJS89jidvkYG3EqKAkgLKog/TQReCq13
HWrF8cMck+Rf2K8k26q/RNZaA9ZUKjLExzz8lsWmd2C7rvkGLrlxnzxz0gGyNR3Q
KK74vcPhqeABt2GSkHtEXZFFA9IVVzwlRWK3e0S+mVQnZVjNL+cBPn3/hZHMLesB
CucyYZv+DxvT+JkYXBkGSw4s3hpABqGym7gdPUIa0q4rbBFG6xP5sLLBG4yru8vV
2dyCMmFqRuxpT49uNfyQ6Vj+dobN6qHnP/9NwfzOixXYBHXR6LBqb/M+iCiJaaIn
uiRLHwkQFPrqY0SOHfkQpA//W51vj8meuz7snRO+vZFcjLneFFzqfh1Jdz8IqDpO
CkI5pBJmi8e0oSe6r68MkahiQLlYPwm7d+sjHvJhPWipNKWq/uwCgBs+Ac1lpPXR
MwLbrZukcLMYlLmb2MrCKmjcMt0BZsZKBNYL3a3X9nHgwXdeqFYS4WQDMCCc09lz
9YqfdoEsqRO4qN7D0hFqnwjOzb34ixZ6UO8a8ekY9QKxAgWc9fJWGMg6Pjdg4qsK
nqymOIAdGVOJdoRM46wKGVBvbsF2gNfQU4XyzgJo5vHGFwJm6EoSnODlL5e2wsQh
uN1oqBt/8ef/plloMEqVBweUBATqSqjRF6IhhYJvWVuQHQL1p1vnV9FebiVj34ir
Z8ID+o0AnTJcclbUcDwannGJ0cuDcPhk/v/ahVuoMERCi12qnMBo5B/e6Omyh1yB
4pbf4GATGGQipDQG75eC/kP2GQEqJP5WYN0Ar8Le/AA/2xyL7upW0yIByyXCwGEb
JRwEgU3+bPyu58bFt8Pftit6J7rA3oBVVMOPrYH5eZwRaj5m2RptwKGL6BfHnhNv
ZqmCq9EBGX6L1NI0xHMjEFfXJ8jU01XdfG8nCqkwqsHwslXLhqjJphfHcx89YwbV
/15GCuURAv1cKe/7277sOhcvP/QpQqSWgvYExHw8PeFJcTYtF2NrRgNwcQsWS1Rj
gXa5Ag0EY+0zcAEQANC5N6kSfezuucAgi+X3BD+MT37mxQyvICSggEJf1LDSmy0+
bnvD7setL8CP9etTA2fcVNYKI1oboMyhoCnsRP2jDdv1iXOI/hZg4wSb/D1yUkae
fUpxv3Wuci2QKavH2MfraDD7BFMbsQeMcHtn4Rk216T6jndZHnzT1Ih7iX0XeQPb
li5fojOiZssgWAVT4HPXFCJB6lI35Hjp35oRYwrtMmu5INinZ79n9h1igGtt1ItZ
b7rQKNd772Jxcn4UU71ovORSL/xT5i5sxZ+evQOxkpqUAokMOFaoHcOXLmA1NsFv
yryXHK4Ioq9ap2jKlLTWkJWjua9JZ4AmKhbvT8X4ELxIKSCAdJKAWP8ZHbXNu5MD
aznyzZQLxSO7uFvu356De75mI5iohZNj5wB5Wju71pBiorTKVj4+iJ4e+xVIzFdG
hFC0DehNcl2t9w/y8qHwIQ1yUAjXHLXq0/2jsVeH6bU5q/MsgvUP1jcFe0eyOpxy
CDvyFdzZFbI57TnB/fvcZTRZ5ewXMFpH8gzuoFzAjUAP95UjYKgaGdrNPNIy28Ii
4zhvdghei2+n9jgiMfcGQg8lyfH5yF0vWWWynX0KcJsRwEZoL2EauVdwq4PcYOoU
pQFhpcreCjD4LdZ4yRU4InbhcUogXjrQ9Dz01TbPmQD5b5iso21bCEFBXrhzABEB
AAGJAjwEGAEKACYCGwwWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwg
hwAKCRAU+upjRI4d+X/XD/sH5xvHPfTJq52v8weFmB52up+DzqG2lyhGdoUQ1Muw
dRDLTLXLJrFdfpoOo7/j4Scr0rdc7/dpCn0DLcPuCoPxu+SkjEnVehFmZrGSv7Ga
x9dHr3DBh42fdlX/U/EnDuyosY0JU1gNF2/6FIA+bTTOFE3RxfN906RjslYQDjMZ
UAlSeLYHOZofdltI0YIr32vrxgdWQGZXPxU4XusDUc0z163OO+TGg7iUNWFZP5Qj
ubM7e0YbDX0NPIshk8us99YJmrWnhaix1/W5ryO3DXiGaQ7XFi9u7QofRqvRIctg
QXavdepkzJow9V9qpMECAJePIuICq7rm+xy+njjbuF436W7390bfVBwRr+FPADsl
jgQP4KvY5rykss30kheom8wNEbveWkhH5oTfH9b7O4KXJfpfJzrlgOWp2BD9JL8t
/M4HvFXTr2a75H/QbHK5OFrZeGATuv9OTxv7EZvnrPXU+DYTFldpu7TrNNqKCoj3
ZyXmc3Hhg5kskDhfHJppaeOayuhMOpT3ud1MFzROY5SLVIH8rBR12KUgsCUYQcGs
Iy0+0QvEGkjb4cAH1NK3VlbqVNsy1RmqRt2B28R2ueewDfTOoqkzt4MmzLqTdnAx
mTqmHmkEKhEf3K4MRNUPO2yieUg2COk5l6x9HhAnoxxeOZrTmcMsPY/UViG2HEPm
ybkCDQRj7TRDARAA9DZuKdfKq4Bs2+NwxC0aplljWOl8VIsEVg+Q8agD7/HU6/b6
Dry0njtWybn2x6Axf/nUdeOC01Fi1lmht/fpj6mRkgAvd/V6P10xnsUoykPSDSTh
P25MFFGW3JAA82bwdJ4AJpEQvTZG2nTb3237vlBiI1qHQrac8GYkju2O4UfySRN6
7cyi7bMf2pjWBBOEhaNy4b6CMDsb32P/N5J7sTE/TXgrS+u4ITIgjzSrkUkh5Z+B
8QVRa7xPIDZJdvZWTEXWu5fgRPZvxbr154GIkWJkFzlDoB1UcO56/uzRUuKhEV6o
HW3LMUuWdPMjpHpq8hrL0G2rDniJFUtbDFzHdZK1LUU3T2BJM8rjI3D/euph+IDT
27vl5qo72zCYE/iKzx4FMLZcQvx1kUAxkPX8l+dzZEwKeRIIpFDxQvatRtl+z0bM
jbkpDb+Yjv66sC4dYRpgTTGX6rok0PWHR3IxDNzyf2j8zQ4LFJ+rVBM1GjGSt6mG
j9TeL8CVeiSp4SuJ7I/FJVPHsKb50m+BDzeB31qTydNqh2kKr0DVAUa+TUsCr7e0
OYr8WE2adJcRXIW0qw50xXF+W7/05GqSCVD0dpeOUdBTQTsSkQmM3/0hcj9aVo9e
UDCM9RF0WRqiDAoHzJFfg+ztamkQI5HO6CklC4Ok22qrHRf6HDNYSuT6QFkAEQEA
AYkCPQQYAQoAJwMbIAQWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwf
tAAKCRAU+upjRI4d+Y+cD/9yllG6uo934pcHNsVppZBfREFwSc8ywlbosCuSVpay
PjSqgrWwDrnqrsk0F2kUdC6rR3BIcXbn+lA9KqylH+cCXAJCkh8EDq6TlQ7Lt5EV
w1U0MAMXOyxPwDymQ/BO+iDyjXWkRRYgbF5XiFhCfGeuKyhkhACisAgNZ1uA1P5k
0SJYc14YfEhQkB46Y20SpfVHRsQ46FyNB6GHbmTmfoO8La8VTh++7GBdh85HfvkG
VNQ3wpi5oXsOLN9+MJOezc0XsW2LQsKQj1/J7QKzGh+lxN5cemsA5aqPzh8dyxeT
0lYRFp4AHkimqGUomVpRkbegMIPxXqOE+ZAmsddErw0UtmrKxcmMptOJwNgYzEgu
++2vtqerL/NYp+wsdcWaBjCz2F3NiwHgNli7NSB/FPwucZZ5gN5C4SnmeFzrGdHg
Oy+tQUN6ayQKljHeBO7CjMlsFNo/dcVrEMa1ShxBMqlj/6ivoEhktLz0Nru4FwNU
xE5SJYDYfpjD7Ws8y4LoXgWXjFHrMO6N9GzqLN/e8LT7I+w4ps2MrgJ8QSrelmQ3
rjkxp3uWp5v2lqy4rLfpi9iB6zIAeoN2eU1yOM9joxOYMxKYaYeYyP1Mm90wFol8
LcTSaN+tVniPddBiL6zvsGBEMbCR9XN3EQ+mErbuw5ovWBOCrr+dvN3FxvD11y4J
7w==
=mXYP
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -1,51 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQENBFZu2zwBCADfatenjH3cvhlU6AeInvp4R0JmPBG942aghFj1Qh57smRcO5Bv
y9mqrX3UDdmVvu58V3k1k9/GzPnAG1t+c7ohdymv/AMuNY4pE2sfxx7bX+mncTHX
5wthipn8kTNm4WjREjCJM1Bm5sozzEZetED3+0/dWlnHl8b38evnLsD+WbSrDPVp
o6M6Eg9IfMwTfcXzdmLmSnGolBWDQ9i1a0x0r3o+sDW5UTnr7jVP+zILcnOZ1Ewl
Rn9OJ4Qg3ULM7WTMDYpKH4BO7RLR3aJgmsFAHp17vgUnzzFBZ10MCS3UOyUNoyph
xo3belf7Q9nrHcSNbqSeQuBnW/vafAZUreAlABEBAAG0IkVlbGNvIERvbHN0cmEg
PGVkb2xzdHJhQGdtYWlsLmNvbT6JATwEEwEIACYCGyMHCwkIBwMCAQYVCAIJCgsE
FgIDAQIeAQIXgAUCVm7etAIZAQAKCRCBcLRybXGY3q51B/96qt41tmcDSzrj/UTl
O6rErfW5zFvVsJTZ95Duwu87t/DVhw5lKBQcjALqVddufw1nMzyN/tSOMVDW8xe4
wMEdcU4+QAMzNX80enuyinsw1glxfLcK0+VbTvqNIfw0sG3MjPqNs6cK2VRfMHK4
paJjytBVICszNX9TfjLyIpKKoSSo1vqnT47LDZ5GIMy7l9Cs2sO/rqQHSPcR79yz
8m8tbHpDDEMZmJeklckKP2QoiqnHiIvlisDxLclYnUmNaPdaN/f++qZz5Yqvu1n+
sNUBA5eLaZH64Uy2SwtABxO3JPJ8nQ2+SFZ7ocFm4Gcdv4aM+Ura9S6fvM91tEJp
yAQOiQE5BBMBCAAjBQJWbts8AhsjBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AA
CgkQgXC0cm1xmN6sIAgAielxO8zJREqEkA2xudg/o4e9ZlNZ3X1NvY8OzJH/qlB2
SmwKqwifhtbC1K0uavXA7eaxdtd2zrI+Yq7IooUyv7juMjHTZhLcFbR5iVkQ4Mfp
JmeHXJ/ChYKxD5mMj/C3WbCZ91oCSNZ6Iyi5fvQj/691OC4q+y/2NEUcOI8D8cw8
XKHbKtceFYc+nZmdOv3ZZrNTSN/kszGViNNLKgnpPdDVPtLp+vjXtbmitiFG2HL/
WfbJ+3Gh2Yr1Vy3O9dWKH++e1AmIv7WWqmUjRFVpqC/wr7/BLaScWT8WKF5vkshU
gq8Ez1/cuizsgs3wQIZWgXKQK5njvwnbKg+Zmh/uGbQmRWVsY28gRG9sc3RyYSA8
ZWVsY28uZG9sc3RyYUB0d2VhZy5pbz6JAU4EEwEIADgWIQS1QdVTAScOC88Vyl2B
cLRybXGY3gUCXELt4gIbIwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCBcLRy
bXGY3ujFCADfS5D1xHU8KH6TpqgssSggYVq62Wwn/Ga+4XPPetM+ajcXagyH6SwB
mxlHICcnv9xC93ryiTI10P1ADJl+aBsI66wEdHBU+ty4RTDy4JZNUPtmRCk9LhSc
mtUO3ry/wtWkRLdJxP49hg7BbQvWoU0M6WODp7SJjPKPWNX64mzHBeOuy+DqGCbM
lpGNCvW8ahU/ewbm7+xwWmzqLDoWzXjHsdF4QdzMVM/vkAgWEP4y0wEqFASzIYaR
GNEkBWU4OQVq5Bdm9+wWWAgsbM0FJAQl0GDqnz4QxWzxxCAAXdbh9F5ffafWYsA9
bise4ZQLkvYo6iUnrcFm4dtZbT8iL3gptCtFZWxjbyBEb2xzdHJhIDxlZWxjby5k
b2xzdHJhQGxvZ2ljYmxveC5jb20+iQE5BBMBCAAjBQJWbt6nAhsjBwsJCAcDAgEG
FQgCCQoLBBYCAwECHgECF4AACgkQgXC0cm1xmN4b/wf8DApMV/jSPEpibekrUPQu
Ye3Z8cxBQuRm/nOPowtPEH/ShAevrCdRiob2nuEZWNoqZ2e5/+6ud07Hs9bslvco
cDv1jeY1dof1idxfKhH3kfSpuD2XJhuzQBxBqOrIlCS/rdnW+Y9wOGD7+bs9QpcA
IyAeQGLLkfggAxaGYQ2Aev8pS7i3a/+lOWbFhcTe02I49KemCOJqBorG5FfILLNr
DjO3EoutNGpuz6rZvc/BlymphWBoAdUmxgoObr7NYWgw9pI8WeE6C7bbSOO7p5aQ
spWXU7Hm17DkzsVDpaJlyClllqK+DdKza5oWlBMe/P02jD3Y+0P/2rCCyQQwmH3D
RbkBDQRWbts8AQgA0g556xc08dH5YNEjbCwEt1j+XoRnV4+GfbSJIXOl9joIgzRC
4IaijvL8+4biWvX7HiybfvBKto0XB1AWLZRC3jWKX5p74I77UAcrD+VQ/roWQqlJ
BKbiQMlRYEsj/5Xnf72G90IP4DAFKvNl+rLChe+jUySA91BCtrYoP75Sw1BE9Cyz
xEtm4WUzKAJdXI+ZTBttA2Nbqy+GSuzBs7fSKDwREJaZmVrosvmns+pQVG4WPWf4
0l4mPguDQmZ9wSWZvBDkpG7AgHYDRYRGkMbAGsVfc6cScN2VsSTa6cbeeAEowKxM
qx9RbY3WOq6aKAm0qDvow1nl7WwXwe8K0wQxfQARAQABiQEfBBgBCAAJBQJWbts8
AhsMAAoJEIFwtHJtcZjeuAAH/0YNz2Qe1IAEO5oqEZNFOccL4KxVPrBhWUen83/b
C6PjOnOqv6q5ztAcms88WIKxBlfzIfq+dzJcbKVS/H7TEXgcaC+7EYW8sJVEsipN
BtEZ3LQNJ5coDjm7WZygniah1lfXNuiritAXduK5FWNNndqGArEaeZ8Shzdo/Uyi
b9lOsBIL6xc2ZcnX5f+rTu02LCEtEb0FwCycZLEWYf8hG4k8uttIOZOC+CLk/k8d
kBmPikMwUVTTV0CdT1cemQKdTaoAaK+kurF6FYXwcnjhRlHrisSt/tVMEwTw4LUM
3MYf6qfjjvE4HlDwZal8th7ccoQp/flfJIuRv85xCcKK+PI=
=u5cX
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -1,13 +0,0 @@
# Maintainer GPG Keys
Release tags are signed by members of the [Nix maintainer team](https://nixos.org/community/teams/nix/) as part of the [release process](../release-process.md). This directory contains the public GPG keys used for signing.
## Keys
- **Eelco Dolstra**
GPG Fingerprint: `B541 D553 0127 0E0B CF15 CA5D 8170 B472 6D71 98DE`
- **Sergei Zimmerman**
GPG Fingerprint: [`158A 6F53 0EA2 02E5 F651 6113 14FA EA63 448E 1DF9`](https://keys.openpgp.org/vks/v1/by-fingerprint/158A6F530EA202E5F651611314FAEA63448E1DF9)
<!-- TODO: Add keys for other Nix team members -->

View File

@@ -5,11 +5,11 @@
The release process is intended to create the following for each
release:
* A signed Git tag (public keys in `maintainers/keys/`)
* A Git tag
* Binary tarballs in https://releases.nixos.org/?prefix=nix/
* Docker images (arm64 and amd64 variants, uploaded to DockerHub and GHCR)
* Docker images
* Closures in https://cache.nixos.org
@@ -104,17 +104,21 @@ release:
evaluation ID (e.g. `1780832` in
`https://hydra.nixos.org/eval/1780832`).
* Tag the release:
* Tag the release and upload the release artifacts to
[`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/):
```console
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
```
Note: `IS_LATEST=1` causes the `latest-release` branch to be
force-updated. This is used by the `nixos.org` website to get the
[latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/).
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
TODO: This script requires the right AWS credentials. Document.
TODO: This script currently requires a
`/home/eelco/Dev/nix-pristine`.
TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
@@ -177,18 +181,16 @@ release:
* Wait for the desired evaluation of the maintenance jobset to finish
building.
* Tag the release
* Run
```console
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
```
Omit `IS_LATEST=1` when creating a point release that is not on the
most recent stable branch. This prevents `nixos.org` to going back
to an older release.
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
* Bump the version number of the release branch as above (e.g. to
`2.12.2`).

View File

@@ -1,8 +1,7 @@
#! /usr/bin/env nix-shell
#! nix-shell -i perl -p awscli2 perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 perlPackages.GetoptLongDescriptive gnupg1
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
use strict;
use Getopt::Long::Descriptive;
use Data::Dumper;
use File::Basename;
use File::Path;
@@ -14,30 +13,7 @@ use Net::Amazon::S3;
delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
my ($opt, $usage) = describe_options(
'%c %o <eval-id>',
[ 'skip-docker', 'Skip Docker image upload' ],
[ 'skip-git', 'Skip Git tagging' ],
[ 'skip-s3', 'Skip S3 upload' ],
[ 'docker-owner=s', 'Docker image owner', { default => 'nixos/nix' } ],
[ 'project-root=s', 'Pristine git repository path' ],
[ 's3-endpoint=s', 'Custom S3 endpoint' ],
[ 's3-host=s', 'S3 host', { default => 's3-eu-west-1.amazonaws.com' } ],
[],
[ 'help|h', 'Show this help message', { shortcircuit => 1 } ],
[],
[ 'Environment variables:' ],
[ 'AWS_ACCESS_KEY_ID' ],
[ 'AWS_SECRET_ACCESS_KEY' ],
[ 'AWS_SESSION_TOKEN For OIDC' ],
[ 'IS_LATEST Set to "1" to mark as latest release' ],
);
print($usage->text), exit if $opt->help;
my $evalId = $ARGV[0] or do { print STDERR $usage->text; exit 1 };
die "--project-root is required unless --skip-git is specified\n" unless $opt->skip_git || $opt->project_root;
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
my $releasesBucketName = "nix-releases";
my $channelsBucketName = "nix-channels";
@@ -86,38 +62,25 @@ File::Path::make_path($narCache);
my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'};
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'};
my $aws_session_token = $ENV{'AWS_SESSION_TOKEN'};
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
my ($s3, $releasesBucket, $s3_channels, $channelsBucket);
my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});
unless ($opt->skip_s3) {
$aws_access_key_id or die "No AWS_ACCESS_KEY_ID given.";
$aws_secret_access_key or die "No AWS_SECRET_ACCESS_KEY given.";
my $releasesBucket = $s3->bucket($releasesBucketName) or die;
$s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
retry => 1,
host => $opt->s3_host,
secure => ($opt->s3_endpoint && $opt->s3_endpoint =~ /^http:/) ? 0 : 1,
});
my $s3_us = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
});
$releasesBucket = $s3->bucket($releasesBucketName) or die;
$s3_channels = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
retry => 1,
$opt->s3_endpoint ? (host => $opt->s3_host) : (),
$opt->s3_endpoint ? (secure => ($opt->s3_endpoint =~ /^http:/) ? 0 : 1) : (),
});
$channelsBucket = $s3_channels->bucket($channelsBucketName) or die;
}
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
sub getStorePath {
my ($jobName, $output) = @_;
@@ -152,12 +115,11 @@ sub copyManual {
File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1});
}
my $awsEndpoint = $opt->s3_endpoint ? "--endpoint-url " . $opt->s3_endpoint : "";
system("aws $awsEndpoint s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
or die "syncing manual to S3\n";
}
copyManual unless $opt->skip_s3;
copyManual;
sub downloadFile {
my ($jobName, $productNr, $dstName) = @_;
@@ -196,12 +158,30 @@ sub downloadFile {
return $sha256_expected;
}
# Upload docker images.
downloadFile("binaryTarball.i686-linux", "1");
downloadFile("binaryTarball.x86_64-linux", "1");
downloadFile("binaryTarball.aarch64-linux", "1");
downloadFile("binaryTarball.x86_64-darwin", "1");
downloadFile("binaryTarball.aarch64-darwin", "1");
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
};
warn "$@" if $@;
downloadFile("installerScript", "1");
# Upload docker images to dockerhub.
my $dockerManifest = "";
my $dockerManifestLatest = "";
my $haveDocker = 0;
unless ($opt->skip_docker) {
for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
my $system = $platforms->[0];
my $dockerPlatform = $platforms->[1];
@@ -215,8 +195,8 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
print STDERR "loading docker image for $dockerPlatform...\n";
system("docker load -i $tmpDir/$fn") == 0 or die;
my $tag = $opt->docker_owner . ":$version-$dockerPlatform";
my $latestTag = $opt->docker_owner . ":latest-$dockerPlatform";
my $tag = "nixos/nix:$version-$dockerPlatform";
my $latestTag = "nixos/nix:latest-$dockerPlatform";
print STDERR "tagging $version docker image for $dockerPlatform...\n";
system("docker tag nix:$version $tag") == 0 or die;
@@ -239,94 +219,68 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
}
if ($haveDocker) {
my $dockerOwner = $opt->docker_owner;
print STDERR "creating multi-platform docker manifest...\n";
system("docker manifest rm $dockerOwner:$version");
system("docker manifest create $dockerOwner:$version $dockerManifest") == 0 or die;
system("docker manifest rm nixos/nix:$version");
system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
if ($isLatest) {
print STDERR "creating latest multi-platform docker manifest...\n";
system("docker manifest rm $dockerOwner:latest");
system("docker manifest create $dockerOwner:latest $dockerManifestLatest") == 0 or die;
system("docker manifest rm nixos/nix:latest");
system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
}
print STDERR "pushing multi-platform docker manifest...\n";
system("docker manifest push $dockerOwner:$version") == 0 or die;
system("docker manifest push nixos/nix:$version") == 0 or die;
if ($isLatest) {
print STDERR "pushing latest multi-platform docker manifest...\n";
system("docker manifest push $dockerOwner:latest") == 0 or die;
system("docker manifest push nixos/nix:latest") == 0 or die;
}
}
}
# Upload nix-fallback-paths.nix.
write_file("$tmpDir/fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
"}\n");
# Upload release files to S3.
unless ($opt->skip_s3) {
downloadFile("binaryTarball.i686-linux", "1");
downloadFile("binaryTarball.x86_64-linux", "1");
downloadFile("binaryTarball.aarch64-linux", "1");
downloadFile("binaryTarball.x86_64-darwin", "1");
downloadFile("binaryTarball.aarch64-darwin", "1");
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
};
warn "$@" if $@;
downloadFile("installerScript", "1");
for my $fn (glob "$tmpDir/*") {
my $name = basename($fn);
next if $name eq "manual";
my $dstKey = "$releaseDir/" . $name;
unless (defined $releasesBucket->head_key($dstKey)) {
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
# Upload nix-fallback-paths.nix.
write_file("$tmpDir/fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
"}\n");
my $configuration = ();
$configuration->{content_type} = "application/octet-stream";
for my $fn (glob "$tmpDir/*") {
my $name = basename($fn);
next if $name eq "manual";
my $dstKey = "$releaseDir/" . $name;
unless (defined $releasesBucket->head_key($dstKey)) {
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
my $configuration = ();
$configuration->{content_type} = "application/octet-stream";
if ($fn =~ /.sha256|install|\.nix$/) {
$configuration->{content_type} = "text/plain";
}
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
if ($fn =~ /.sha256|install|\.nix$/) {
$configuration->{content_type} = "text/plain";
}
}
# Update the "latest" symlink.
$channelsBucket->add_key(
"nix-latest/install", "",
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
or die $channelsBucket->err . ": " . $channelsBucket->errstr
if $isLatest;
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
}
}
# Update the "latest" symlink.
$channelsBucket->add_key(
"nix-latest/install", "",
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
or die $channelsBucket->err . ": " . $channelsBucket->errstr
if $isLatest;
# Tag the release in Git.
unless ($opt->skip_git) {
chdir($opt->project_root) or die "Cannot chdir to " . $opt->project_root . ": $!";
system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
system("git push origin refs/tags/$version") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
}
chdir("/home/eelco/Dev/nix-pristine") or die;
system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
system("git push --tags") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
File::Path::remove_tree($narCache, {safe => 1});
File::Path::remove_tree($tmpDir, {safe => 1});

View File

@@ -148,15 +148,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
isInternal =
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
activeComponentNames = lib.listToAttrs (
map (c: {
name = c.pname or c.name;
value = null;
}) activeComponents
);
isActiveComponent = name: activeComponentNames ? ${name};
in
{
pname = "shell-for-nix";
@@ -199,19 +190,27 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
}
);
small = finalAttrs.finalPackage.withActiveComponents (
c:
lib.intersectAttrs (lib.genAttrs [
"nix-cli"
"nix-util-tests"
"nix-store-tests"
"nix-expr-tests"
"nix-fetchers-tests"
"nix-flake-tests"
"nix-functional-tests"
"nix-perl-bindings"
] (_: null)) c
);
small =
(finalAttrs.finalPackage.withActiveComponents (
c:
lib.intersectAttrs (lib.genAttrs [
"nix-cli"
"nix-util-tests"
"nix-store-tests"
"nix-expr-tests"
"nix-fetchers-tests"
"nix-flake-tests"
"nix-functional-tests"
"nix-perl-bindings"
] (_: null)) c
)).overrideAttrs
(o: {
mesonFlags = o.mesonFlags ++ [
# TODO: infer from activeComponents or vice versa
"-Dkaitai-struct-checks=false"
"-Djson-schema-checks=false"
];
});
};
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
@@ -276,33 +275,21 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
dontUseCmakeConfigure = true;
mesonFlags = [
(lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks"))
(lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
]
++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
++ lib.optionals havePerl (
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
)
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
mesonFlags =
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
++ lib.optionals havePerl (
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
)
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
nativeBuildInputs =
let
inputs =
dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (
lib.lists.concatMap (
# Nix manual has a build-time dependency on nix, but we
# don't want to do a native build just to enter the ross
# dev shell.
#
# TODO: think of a more principled fix for this.
c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs
) activeComponents
)
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
)
++ lib.optional (
!buildCanExecuteHost
@@ -318,8 +305,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
pkgs.buildPackages.nixfmt-rfc-style
pkgs.buildPackages.shellcheck
pkgs.buildPackages.include-what-you-use
pkgs.buildPackages.gdb
]
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
lib.hiPrio pkgs.buildPackages.clang-tools
)
@@ -335,13 +322,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
)
);
buildInputs =
# TODO change Nixpkgs to mark gbenchmark as building on Windows
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
++ lib.optional havePerl pkgs.perl;
buildInputs = [
pkgs.gbenchmark
]
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
++ lib.optional havePerl pkgs.perl;
propagatedBuildInputs = dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)

View File

@@ -1427,11 +1427,7 @@ namespace fetchers {
ref<GitRepo> Settings::getTarballCache() const
{
/* v1: Had either only loose objects or thin packfiles referring to loose objects
* v2: Must have only packfiles with no loose objects. Should get repacked periodically
* for optimal packfiles.
*/
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2";
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
return GitRepo::openRepo(repoDir, /*create=*/true, /*bare=*/true, /*packfilesOnly=*/true);
}

View File

@@ -4,20 +4,15 @@
# include <aws/crt/Types.h>
# include "nix/store/s3-url.hh"
# include "nix/util/finally.hh"
# include "nix/util/logging.hh"
# include "nix/util/url.hh"
# include "nix/util/util.hh"
# include <aws/crt/Api.h>
# include <aws/crt/auth/Credentials.h>
# include <aws/crt/io/Bootstrap.h>
// C library headers for SSO provider support
# include <aws/auth/credentials.h>
// C library headers for custom logging
# include <aws/common/logging.h>
# include <cstdarg>
# include <boost/unordered/concurrent_flat_map.hpp>
# include <chrono>
@@ -35,141 +30,6 @@ AwsAuthError::AwsAuthError(int errorCode)
namespace {
/**
* Map AWS log level to Nix verbosity.
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
*
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
* considers "info" includes low-level details like "Initializing epoll" and
* "Starting event-loop thread". What it considers "errors" includes expected
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
*
* To avoid spamming users, we only show FATAL at default verbosity. Everything
* else requires -vvvvv (lvlDebug) or higher to see.
*/
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
{
switch (level) {
case AWS_LL_FATAL:
return lvlError;
case AWS_LL_ERROR:
case AWS_LL_WARN:
case AWS_LL_INFO:
return lvlDebug;
case AWS_LL_DEBUG:
case AWS_LL_TRACE:
return lvlVomit;
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
case AWS_LL_NONE:
case AWS_LL_COUNT:
return lvlDebug;
}
unreachable();
}
/**
* Custom AWS logger that routes logs through Nix's logging infrastructure.
*
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
* or filename-based logging. The underlying C library supports custom loggers
* via aws_logger struct with a vtable containing callback functions.
*/
static int nixAwsLoggerLog(
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
{
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
if (nixLevel > verbosity)
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
va_list args;
va_start(args, format);
std::array<char, 4096> buffer{};
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
va_end(args);
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
return AWS_OP_SUCCESS;
const char * subjectName = aws_log_subject_name(subject);
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
return AWS_OP_SUCCESS;
}
/**
* Get current log level for a subject - determines which messages will be logged.
* Must be consistent with awsLogLevelToVerbosity mapping.
*/
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
{
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
if (verbosity >= lvlVomit)
return AWS_LL_TRACE;
if (verbosity >= lvlDebug)
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
return AWS_LL_FATAL;
}
static void initialiseAwsLogger()
{
static std::once_flag initialised; /* aws_logger_set must only be called once */
std::call_once(initialised, []() {
static aws_logger_vtable nixAwsLoggerVtable = {
.log = nixAwsLoggerLog,
.get_log_level = nixAwsLoggerGetLevel,
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
.set_log_level = nullptr,
};
static aws_logger nixAwsLogger = {
.vtable = &nixAwsLoggerVtable,
.allocator = nullptr,
.p_impl = nullptr,
};
aws_logger_set(&nixAwsLogger);
});
}
/**
* Helper function to wrap a C credentials provider in the C++ interface.
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
*/
static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createWrappedProvider(
aws_credentials_provider * rawProvider, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
{
if (rawProvider == nullptr) {
return nullptr;
}
auto provider = Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider>(allocator, rawProvider, allocator);
return std::static_pointer_cast<Aws::Crt::Auth::ICredentialsProvider>(provider);
}
/**
* Create an SSO credentials provider using the C library directly.
* The C++ wrapper doesn't expose SSO, so we call the C library and wrap the result.
* Returns nullptr if SSO provider creation fails (e.g., profile doesn't have SSO config).
*/
static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createSSOProvider(
const std::string & profileName,
Aws::Crt::Io::ClientBootstrap * bootstrap,
Aws::Crt::Io::TlsContext * tlsContext,
Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
{
aws_credentials_provider_sso_options options;
AWS_ZERO_STRUCT(options);
options.bootstrap = bootstrap->GetUnderlyingHandle();
options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr;
if (!profileName.empty()) {
options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str());
}
// Create the SSO provider - will return nullptr if SSO isn't configured for this profile
// createWrappedProvider handles nullptr gracefully
return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator);
}
static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
{
if (!provider || !provider->IsValid()) {
@@ -219,25 +79,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
public:
AwsCredentialProviderImpl()
{
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
initialiseAwsLogger();
// Create a shared TLS context for SSO (required for HTTPS connections)
auto allocator = Aws::Crt::ApiAllocator();
auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator);
tlsContext =
std::make_shared<Aws::Crt::Io::TlsContext>(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator);
if (!tlsContext || !*tlsContext) {
warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable");
tlsContext = nullptr;
}
// Get bootstrap (lives as long as apiHandle)
bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
if (!bootstrap) {
throw AwsAuthError("failed to create AWS client bootstrap");
// Map Nix's verbosity to AWS CRT log level
Aws::Crt::LogLevel logLevel;
if (verbosity >= lvlVomit) {
logLevel = Aws::Crt::LogLevel::Trace;
} else if (verbosity >= lvlDebug) {
logLevel = Aws::Crt::LogLevel::Debug;
} else if (verbosity >= lvlChatty) {
logLevel = Aws::Crt::LogLevel::Info;
} else {
logLevel = Aws::Crt::LogLevel::Warn;
}
apiHandle.InitializeLogging(logLevel, stderr);
}
AwsCredentials getCredentialsRaw(const std::string & profile);
@@ -258,8 +111,6 @@ public:
private:
Aws::Crt::ApiHandle apiHandle;
std::shared_ptr<Aws::Crt::Io::TlsContext> tlsContext;
Aws::Crt::Io::ClientBootstrap * bootstrap;
boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
credentialProviderCache;
};
@@ -267,58 +118,23 @@ private:
std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
{
// profileDisplayName is only used for debug logging - SDK uses its default profile
// when ProfileNameOverride is not set
const char * profileDisplayName = profile.empty() ? "(default)" : profile.c_str();
debug(
"[pid=%d] creating new AWS credential provider for profile '%s'",
getpid(),
profile.empty() ? "(default)" : profile.c_str());
debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName);
// Build a custom credential chain: Environment → SSO → Profile → IMDS
// This works for both default and named profiles, ensuring consistent behavior
// including SSO support and proper TLS context for STS-based role assumption.
Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig;
auto allocator = Aws::Crt::ApiAllocator();
auto addProviderToChain = [&](std::string_view name, auto createProvider) {
if (auto provider = createProvider()) {
chainConfig.Providers.push_back(provider);
debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName);
} else {
debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName);
}
};
// 1. Environment variables (highest priority)
addProviderToChain("Environment", [&]() {
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderEnvironment(allocator);
});
// 2. SSO provider (try it, will fail gracefully if not configured)
if (tlsContext) {
addProviderToChain("SSO", [&]() { return createSSOProvider(profile, bootstrap, tlsContext.get(), allocator); });
} else {
debug("Skipped AWS SSO Credential Provider for profile '%s': TLS context unavailable", profileDisplayName);
if (profile.empty()) {
Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
}
// 3. Profile provider (for static credentials and role_arn/source_profile with STS)
addProviderToChain("Profile", [&]() {
Aws::Crt::Auth::CredentialsProviderProfileConfig profileConfig;
profileConfig.Bootstrap = bootstrap;
profileConfig.TlsContext = tlsContext.get();
if (!profile.empty()) {
profileConfig.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
}
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator);
});
// 4. IMDS provider (for EC2 instances, lowest priority)
addProviderToChain("IMDS", [&]() {
Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig;
imdsConfig.Bootstrap = bootstrap;
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator);
});
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator);
Aws::Crt::Auth::CredentialsProviderProfileConfig config;
config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
// This is safe because the underlying C library will copy this string
// c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
}
AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)

View File

@@ -563,7 +563,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
{
DerivationBuildingGoal & goal;
DerivationBuildingGoalCallbacks(DerivationBuildingGoal & goal)
DerivationBuildingGoalCallbacks(
DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
: goal{goal}
{
}
@@ -631,15 +632,15 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
/* If we have to wait and retry (see below), then `builder` will
already be created, so we don't need to create it again. */
builder =
externalBuilder
? makeExternalDerivationBuilder(
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this),
std::move(params),
*externalBuilder)
: makeDerivationBuilder(
*localStoreP, std::make_unique<DerivationBuildingGoalCallbacks>(*this), std::move(params));
builder = externalBuilder ? makeExternalDerivationBuilder(
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
std::move(params),
*externalBuilder)
: makeDerivationBuilder(
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
std::move(params));
}
if (auto builderOutOpt = builder->startBuild()) {
@@ -1175,6 +1176,11 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs)
{
buildResult.inner = BuildResult::Success{
.status = status,
.builtOutputs = std::move(builtOutputs),
};
mcRunningBuilds.reset();
if (status == BuildResult::Success::Built)
@@ -1182,15 +1188,16 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status stat
worker.updateProgress();
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
.builtOutputs = std::move(builtOutputs),
});
return amDone(ecSuccess, std::nullopt);
}
Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
{
buildResult.inner = BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
};
mcRunningBuilds.reset();
if (ex.status == BuildResult::Failure::TimedOut)
@@ -1202,13 +1209,7 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
worker.updateProgress();
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
return amDone(ecFailed, {std::move(ex)});
}
} // namespace nix

View File

@@ -452,6 +452,20 @@ UnkeyedRealisation DerivationGoal::assertPathValidity()
Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, UnkeyedRealisation builtOutput)
{
buildResult.inner = BuildResult::Success{
.status = status,
.builtOutputs = {{
wantedOutput,
{
std::move(builtOutput),
DrvOutput{
.drvHash = outputHash,
.outputName = wantedOutput,
},
},
}},
};
mcExpectedBuilds.reset();
if (status == BuildResult::Success::Built)
@@ -459,24 +473,16 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
worker.updateProgress();
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
.builtOutputs = {{
wantedOutput,
{
std::move(builtOutput),
DrvOutput{
.drvHash = outputHash,
.outputName = wantedOutput,
},
},
}},
});
return amDone(ecSuccess, std::nullopt);
}
Goal::Done DerivationGoal::doneFailure(BuildError ex)
{
buildResult.inner = BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
};
mcExpectedBuilds.reset();
if (ex.status == BuildResult::Failure::TimedOut)
@@ -488,13 +494,7 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
worker.updateProgress();
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
return amDone(ecFailed, {std::move(ex)});
}
} // namespace nix

View File

@@ -98,13 +98,7 @@ Goal::Co DerivationTrampolineGoal::init()
trace("outer load and build derivation");
if (nrFailed != 0) {
co_return doneFailure(
ecFailed,
BuildResult::Failure{
.status = BuildResult::Failure::DependencyFailed,
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
},
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);

View File

@@ -63,18 +63,12 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
std::vector<KeyedBuildResult> results;
results.reserve(state.size());
for (auto & [req, goalPtr] : state) {
/* Goals that were never started or were cancelled have exitCode
ecBusy and a default buildResult with empty errorMsg. Skip them
to avoid reporting spurious failures with empty messages. */
if (goalPtr->exitCode == Goal::ecBusy)
continue;
for (auto & [req, goalPtr] : state)
results.emplace_back(
KeyedBuildResult{
goalPtr->buildResult,
/* .path = */ req,
});
}
return results;
}

View File

@@ -133,19 +133,6 @@ Co Goal::await(Goals new_waitees)
co_return Return{};
}
Goal::Done Goal::doneSuccess(BuildResult::Success success)
{
buildResult.inner = std::move(success);
return amDone(ecSuccess);
}
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
{
assert(result == ecFailed || result == ecNoSubstituters);
buildResult.inner = std::move(failure);
return amDone(result, std::move(ex));
}
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
{
trace("done");

View File

@@ -29,21 +29,20 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
{
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
});
buildResult.inner = BuildResult::Success{
.status = status,
};
return amDone(ecSuccess);
}
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
{
debug(errorMsg);
return Goal::doneFailure(
result,
BuildResult::Failure{
.status = status,
.errorMsg = std::move(errorMsg),
});
buildResult.inner = BuildResult::Failure{
.status = status,
.errorMsg = std::move(errorMsg),
};
return amDone(result);
}
Goal::Co PathSubstitutionGoal::init()

View File

@@ -341,8 +341,12 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
if (parsed) {
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
if (!e || !e->is_object())
if (!e)
return ret;
if (!e->is_object()) {
warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring");
return ret;
}
for (auto & [key, storePathsJson] : getObject(*e)) {
StringSet ss;
flatten(storePathsJson, ss);

View File

@@ -1489,6 +1489,8 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
}
}
static unsigned constexpr expectedJsonVersionDerivation = 4;
void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
{
res = nlohmann::json::object();

View File

@@ -131,16 +131,7 @@ struct curlFileTransfer : public FileTransfer
{
result.urls.push_back(request.uri.to_string());
/* Don't set Accept-Encoding for S3 requests that use AWS SigV4 signing.
curl's SigV4 implementation signs all headers including Accept-Encoding,
but some S3-compatible services (like GCS) modify this header in transit,
causing signature verification to fail.
See https://github.com/NixOS/nix/issues/15019 */
#if NIX_WITH_AWS_AUTH
if (!request.awsSigV4Provider)
#endif
requestHeaders =
curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
if (!request.expectedETag.empty())
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
if (!request.mimeType.empty())
@@ -450,11 +441,6 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
if (request.data) {
// Restart the source to ensure it's at the beginning.
// This is necessary for retries, where the source was
// already consumed by a previous attempt.
request.data->source->restart();
if (request.method == HttpMethod::Post) {
curl_easy_setopt(req, CURLOPT_POST, 1L);
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);

View File

@@ -79,7 +79,7 @@ struct DerivationBuilderParams
*/
const StorePathSet & inputPaths;
const std::map<std::string, InitialOutput> initialOutputs;
const std::map<std::string, InitialOutput> & initialOutputs;
const BuildMode & buildMode;
@@ -189,22 +189,15 @@ struct ExternalBuilder
std::vector<std::string> args;
};
struct DerivationBuilderDeleter
{
void operator()(DerivationBuilder * builder) noexcept;
};
using DerivationBuilderUnique = std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter>;
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
DerivationBuilderUnique makeDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params);
/**
* @param handler Must be chosen such that it supports the given
* derivation.
*/
DerivationBuilderUnique makeExternalDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
LocalStore & store,
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
DerivationBuilderParams params,

View File

@@ -5,7 +5,6 @@
#include "nix/store/parsed-derivations.hh"
#include "nix/store/derivation-options.hh"
#include "nix/store/build/derivation-building-misc.hh"
#include "nix/store/build/derivation-builder.hh"
#include "nix/store/outputs-spec.hh"
#include "nix/store/store-api.hh"
#include "nix/store/pathlocks.hh"
@@ -90,7 +89,7 @@ private:
*/
std::unique_ptr<HookInstance> hook;
DerivationBuilderUnique builder;
std::unique_ptr<DerivationBuilder> builder;
#endif
BuildMode buildMode;

View File

@@ -393,28 +393,9 @@ protected:
* Signals that the goal is done.
* `co_return` the result. If you're not inside a coroutine, you can ignore
* the return value safely.
*
* Prefer using `doneSuccess` or `doneFailure` instead, which ensure
* `buildResult` is set correctly.
*/
Done amDone(ExitCode result, std::optional<Error> ex = {});
/**
* Signals successful completion of the goal.
* Sets `buildResult` and calls `amDone`.
*/
Done doneSuccess(BuildResult::Success success);
/**
* Signals failed completion of the goal.
* Sets `buildResult` and calls `amDone`.
*
* @param result The exit code (ecFailed or ecNoSubstituters)
* @param failure The failure details including status and error message
* @param ex Optional exception to store/log
*/
Done doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex = {});
public:
virtual void cleanup() {}

View File

@@ -586,12 +586,6 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
*/
std::string hashPlaceholder(const OutputNameView outputName);
/**
* The expected JSON version for derivation serialization.
* Used by `nix derivation show` and `nix derivation add`.
*/
constexpr unsigned expectedJsonVersionDerivation = 4;
} // namespace nix
JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)

View File

@@ -1143,7 +1143,7 @@ public:
Setting<std::string> netrcFile{
this,
(nixConfDir / "netrc").string(),
fmt("%s/%s", nixConfDir, "netrc"),
"netrc-file",
R"(
If set to an absolute path to a `netrc` file, Nix uses the HTTP

View File

@@ -160,8 +160,6 @@ if s3_aws_auth.enabled()
deps_other += aws_crt_cpp
aws_c_common = cxx.find_library('aws-c-common', required : true)
deps_other += aws_c_common
aws_c_auth = cxx.find_library('aws-c-auth', required : true)
deps_other += aws_c_auth
endif
configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())

View File

@@ -454,8 +454,6 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
.downloadSize = narInfo ? narInfo->fileSize : 0,
.narSize = info->narSize,
});
break; /* We are done. */
} catch (InvalidPath &) {
} catch (SubstituterDisabled &) {
} catch (Error & e) {

View File

@@ -98,13 +98,10 @@ public:
{
}
/**
* Cleanup code to run when destroying any DerivationBuilderImpl implementation.
*/
void cleanupOnDestruction() noexcept
~DerivationBuilderImpl()
{
/* Careful: we should never ever throw an exception from a
noexcept function. */
destructor. */
try {
killChild();
} catch (...) {
@@ -681,17 +678,17 @@ static void handleChildException(bool sendException)
}
}
static void checkNotWorldWritable(std::filesystem::path path)
static bool checkNotWorldWritable(std::filesystem::path path)
{
while (true) {
auto st = lstat(path);
if (st.st_mode & S_IWOTH)
throw Error("Path %s is world-writable or a symlink. That's not allowed for security.", path);
return false;
if (path == path.parent_path())
break;
path = path.parent_path();
}
return;
return true;
}
std::optional<Descriptor> DerivationBuilderImpl::startBuild()
@@ -713,8 +710,9 @@ std::optional<Descriptor> DerivationBuilderImpl::startBuild()
createDirs(buildDir);
if (buildUser)
checkNotWorldWritable(buildDir);
if (buildUser && !checkNotWorldWritable(buildDir))
throw Error(
"Path %s or a parent directory is world-writable or a symlink. That's not allowed for security.", buildDir);
/* Create a temporary directory where the build will take
place. */
@@ -1965,20 +1963,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
namespace nix {
void DerivationBuilderDeleter::operator()(DerivationBuilder * builder) noexcept
{
if (!builder) /* Idempotent and handles nullptr as any deleter must. */
return;
if (auto builderImpl = dynamic_cast<DerivationBuilderImpl *>(builder))
/* Note that this might call into virtual functions, which we can't do in a destructor of
the DerivationBuilderImpl itself. */
builderImpl->cleanupOnDestruction();
delete builder;
}
std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
{
bool useSandbox = false;
@@ -2029,19 +2014,17 @@ std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuild
throw Error("feature 'uid-range' is only supported in sandboxed builds");
#ifdef __APPLE__
return DerivationBuilderUnique(
new DarwinDerivationBuilder(store, std::move(miscMethods), std::move(params), useSandbox));
return std::make_unique<DarwinDerivationBuilder>(store, std::move(miscMethods), std::move(params), useSandbox);
#elif defined(__linux__)
if (useSandbox)
return DerivationBuilderUnique(
new ChrootLinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
return std::make_unique<ChrootLinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
return DerivationBuilderUnique(new LinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
return std::make_unique<LinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
#else
if (useSandbox)
throw Error("sandboxing builds is not supported on this platform");
return DerivationBuilderUnique(new DerivationBuilderImpl(store, std::move(miscMethods), std::move(params)));
return std::make_unique<DerivationBuilderImpl>(store, std::move(miscMethods), std::move(params));
#endif
}

View File

@@ -106,14 +106,13 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl
}
};
DerivationBuilderUnique makeExternalDerivationBuilder(
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
LocalStore & store,
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
DerivationBuilderParams params,
const ExternalBuilder & handler)
{
return DerivationBuilderUnique(
new ExternalDerivationBuilder(store, std::move(miscMethods), std::move(params), handler));
return std::make_unique<ExternalDerivationBuilder>(store, std::move(miscMethods), std::move(params), handler);
}
} // namespace nix

View File

@@ -16,7 +16,6 @@ R""(
; Allow DNS lookups.
(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))
(allow mach-lookup (global-name "com.apple.SystemConfiguration.DNSConfiguration"))
; Allow access to trustd.
(allow mach-lookup (global-name "com.apple.trustd"))

View File

@@ -713,27 +713,17 @@ AutoCloseFD createAnonymousTempFile()
{
AutoCloseFD fd;
#ifdef O_TMPFILE
static std::atomic_flag tmpfileUnsupported{};
if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ {
/* Use O_EXCL, because the file is never supposed to be linked into filesystem. */
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR | O_EXCL, S_IWUSR | S_IRUSR);
if (!fd) {
/* Not supported by the filesystem or the kernel. */
if (errno == EOPNOTSUPP || errno == EISDIR)
tmpfileUnsupported.test_and_set(); /* Set flag and fall through to createTempFile. */
else
throw SysError("creating anonymous temporary file");
} else {
return fd; /* Successfully created. */
}
}
#endif
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR, S_IWUSR | S_IRUSR);
if (!fd)
throw SysError("creating anonymous temporary file");
#else
auto [fd2, path] = createTempFile("nix-anonymous");
if (!fd2)
throw SysError("creating temporary file '%s'", path);
fd = std::move(fd2);
#ifndef _WIN32
# ifndef _WIN32
unlink(requireCString(path)); /* We only care about the file descriptor. */
# endif
#endif
return fd;
}

View File

@@ -65,7 +65,6 @@ headers = files(
'signals.hh',
'signature/local-keys.hh',
'signature/signer.hh',
'socket.hh',
'sort.hh',
'source-accessor.hh',
'source-path.hh',

View File

@@ -1,61 +0,0 @@
#pragma once
///@file
#include "nix/util/file-descriptor.hh"
#ifdef _WIN32
# include <winsock2.h>
#endif
namespace nix {
/**
* Often we want to use `Descriptor`, but Windows makes a slightly
* stronger file descriptor vs socket distinction, at least at the level
* of C types.
*/
using Socket =
#ifdef _WIN32
SOCKET
#else
int
#endif
;
#ifdef _WIN32
/**
* Windows gives this a different name
*/
# define SHUT_WR SD_SEND
# define SHUT_RDWR SD_BOTH
#endif
/**
* Convert a `Descriptor` to a `Socket`
*
* This is a no-op except on Windows.
*/
static inline Socket toSocket(Descriptor fd)
{
#ifdef _WIN32
return reinterpret_cast<Socket>(fd);
#else
return fd;
#endif
}
/**
* Convert a `Socket` to a `Descriptor`
*
* This is a no-op except on Windows.
*/
static inline Descriptor fromSocket(Socket fd)
{
#ifdef _WIN32
return reinterpret_cast<Descriptor>(fd);
#else
return fd;
#endif
}
} // namespace nix

View File

@@ -3,8 +3,10 @@
#include "nix/util/types.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/socket.hh"
#ifdef _WIN32
# include <winsock2.h>
#endif
#include <unistd.h>
#include <filesystem>
@@ -21,6 +23,55 @@ AutoCloseFD createUnixDomainSocket();
*/
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/**
* Often we want to use `Descriptor`, but Windows makes a slightly
* stronger file descriptor vs socket distinction, at least at the level
* of C types.
*/
using Socket =
#ifdef _WIN32
SOCKET
#else
int
#endif
;
#ifdef _WIN32
/**
* Windows gives this a different name
*/
# define SHUT_WR SD_SEND
# define SHUT_RDWR SD_BOTH
#endif
/**
* Convert a `Socket` to a `Descriptor`
*
* This is a no-op except on Windows.
*/
static inline Socket toSocket(Descriptor fd)
{
#ifdef _WIN32
return reinterpret_cast<Socket>(fd);
#else
return fd;
#endif
}
/**
* Convert a `Socket` to a `Descriptor`
*
* This is a no-op except on Windows.
*/
static inline Descriptor fromSocket(Socket fd)
{
#ifdef _WIN32
return reinterpret_cast<Descriptor>(fd);
#else
return fd;
#endif
}
/**
* Bind a Unix domain socket to a path.
*/

View File

@@ -1,7 +1,6 @@
#include "nix/util/serialise.hh"
#include "nix/util/compression.hh"
#include "nix/util/signals.hh"
#include "nix/util/socket.hh"
#include "nix/util/util.hh"
#include <cstring>
@@ -12,6 +11,7 @@
#ifdef _WIN32
# include <fileapi.h>
# include <winsock2.h>
# include "nix/util/windows-error.hh"
#else
# include <poll.h>
@@ -184,20 +184,20 @@ bool FdSource::hasData()
while (true) {
fd_set fds;
FD_ZERO(&fds);
Socket sock = toSocket(fd);
FD_SET(sock, &fds);
int fd_ = fromDescriptorReadOnly(fd);
FD_SET(fd_, &fds);
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = 0;
auto n = select(sock + 1, &fds, nullptr, nullptr, &timeout);
auto n = select(fd_ + 1, &fds, nullptr, nullptr, &timeout);
if (n < 0) {
if (errno == EINTR)
continue;
throw SysError("polling file descriptor");
}
return FD_ISSET(sock, &fds);
return FD_ISSET(fd, &fds);
}
}

View File

@@ -35,18 +35,14 @@ struct UnionSourceAccessor : SourceAccessor
DirEntries readDirectory(const CanonPath & path) override
{
DirEntries result;
bool exists = false;
for (auto & accessor : accessors) {
auto st = accessor->maybeLstat(path);
if (!st)
continue;
exists = true;
for (auto & entry : accessor->readDirectory(path))
// Don't override entries from previous accessors.
result.insert(entry);
}
if (!exists)
throw FileNotFound("path '%s' does not exist", showPath(path));
return result;
}

View File

@@ -8,12 +8,9 @@
#include <unistd.h>
#include <poll.h>
#if defined(__linux__)
# include <sys/syscall.h> /* pull __NR_* definitions */
#endif
#if defined(__linux__) && defined(__NR_openat2)
# define HAVE_OPENAT2 1
# include <sys/syscall.h>
# include <linux/openat2.h>
#else
# define HAVE_OPENAT2 0
@@ -326,7 +323,7 @@ Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPa
{
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
assert(!path.isRoot());
#if HAVE_OPENAT2
#ifdef __linux__
auto maybeFd = linux::openat2(
dirFd, path.rel_c_str(), flags, static_cast<uint64_t>(mode), RESOLVE_BENEATH | RESOLVE_NO_SYMLINKS);
if (maybeFd) {

View File

@@ -60,11 +60,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON
jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath);
}
printJSON(
nlohmann::json{
{"version", expectedJsonVersionDerivation},
{"derivations", std::move(jsonRoot)},
});
printJSON(jsonRoot);
}
};

View File

@@ -437,23 +437,22 @@ static void forwardStdioConnection(RemoteStore & store)
int from = conn->from.fd;
int to = conn->to.fd;
Socket fromSock = toSocket(from), stdinSock = toSocket(getStandardInput());
auto nfds = std::max(fromSock, stdinSock) + 1;
auto nfds = std::max(from, STDIN_FILENO) + 1;
while (true) {
fd_set fds;
FD_ZERO(&fds);
FD_SET(fromSock, &fds);
FD_SET(stdinSock, &fds);
FD_SET(from, &fds);
FD_SET(STDIN_FILENO, &fds);
if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
throw SysError("waiting for data from client or server");
if (FD_ISSET(fromSock, &fds)) {
if (FD_ISSET(from, &fds)) {
auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
if (res == -1)
throw SysError("splicing data from daemon socket to stdout");
else if (res == 0)
throw EndOfFile("unexpected EOF from daemon socket");
}
if (FD_ISSET(stdinSock, &fds)) {
if (FD_ISSET(STDIN_FILENO, &fds)) {
auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
if (res == -1)
throw SysError("splicing data from stdin to daemon socket");

View File

@@ -158,24 +158,14 @@ printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []'
printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")'
# --keep-going and FOD
if isDaemonNewer "2.34pre"; then
# With the fix, cancelled goals are not reported as failures.
# Use -j1 so only x1 starts and fails; x2, x3, x4 are cancelled.
out="$(nix build -f fod-failing.nix -j1 -L 2>&1)" && status=0 || status=$?
test "$status" = 1
# Only the hash mismatch error for x1. Cancelled goals not reported.
test "$(<<<"$out" grep -cE '^error:')" = 1
# Regression test: error messages should not be empty (end with just "failed:")
<<<"$out" grepQuietInverse -E "^error:.*failed: *$"
else
out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$?
test "$status" = 1
# At minimum, check that x1 is reported as failing
<<<"$out" grepQuiet -E "error:.*-x1"
fi
out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$?
test "$status" = 1
# one "hash mismatch" error, one "build of ... failed"
test "$(<<<"$out" grep -cE '^error:')" = 2
<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'"
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'"
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'"
<<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed"
out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$?
test "$status" = 1
@@ -213,69 +203,3 @@ else
fi
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'"
<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'"
# Regression test: cancelled builds should not be reported as failures
# When fast-fail fails, slow and depends-on-slow are cancelled (not failed).
# Only fast-fail should be reported as a failure.
# Uses fifo for synchronization to ensure deterministic behavior.
# Requires -j2 so slow and fast-fail run concurrently (fifo deadlocks if serialized).
if isDaemonNewer "2.34pre" && canUseSandbox; then
fifoDir="$TEST_ROOT/cancelled-builds-fifo"
mkdir -p "$fifoDir"
mkfifo "$fifoDir/fifo"
chmod a+rw "$fifoDir/fifo"
# When using a separate test store, we need sandbox-paths to access
# the system store (where bash/coreutils live). On NixOS, the test
# uses the system store directly, so this isn't needed (and would
# conflict with input paths).
sandboxPathsArg=()
if ! isTestOnNixOS; then
sandboxPathsArg=(--option sandbox-paths "/nix/store")
fi
out="$(nix flake check ./cancelled-builds --impure -L -j2 \
--option sandbox true \
"${sandboxPathsArg[@]}" \
--option sandbox-build-dir /build-tmp \
--option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \
2>&1)" && status=0 || status=$?
rm -rf "$fifoDir"
test "$status" = 1
# The error should be for fast-fail, not for cancelled goals
<<<"$out" grepQuiet -E "Cannot build.*fast-fail"
# Cancelled goals should NOT appear in error messages (but may appear in "will be built" list)
<<<"$out" grepQuietInverse -E "^error:.*slow"
<<<"$out" grepQuietInverse -E "^error:.*depends-on-slow"
<<<"$out" grepQuietInverse -E "^error:.*depends-on-fail"
# Error messages should not be empty (end with just "failed:")
<<<"$out" grepQuietInverse -E "^error:.*failed: *$"
# Test that nix build follows the same rules (uses a slightly different code path)
mkdir -p "$fifoDir"
mkfifo "$fifoDir/fifo"
chmod a+rw "$fifoDir/fifo"
sandboxPathsArg=()
if ! isTestOnNixOS; then
sandboxPathsArg=(--option sandbox-paths "/nix/store")
fi
system=$(nix eval --raw --impure --expr builtins.currentSystem)
out="$(nix build --impure -L -j2 \
--option sandbox true \
"${sandboxPathsArg[@]}" \
--option sandbox-build-dir /build-tmp \
--option extra-sandbox-paths "/cancelled-builds-fifo=$fifoDir" \
"./cancelled-builds#checks.$system.slow" \
"./cancelled-builds#checks.$system.depends-on-slow" \
"./cancelled-builds#checks.$system.fast-fail" \
"./cancelled-builds#checks.$system.depends-on-fail" \
2>&1)" && status=0 || status=$?
rm -rf "$fifoDir"
test "$status" = 1
# The error should be for fast-fail, not for cancelled goals
<<<"$out" grepQuiet -E "Cannot build.*fast-fail"
# Cancelled goals should NOT appear in error messages
<<<"$out" grepQuietInverse -E "^error:.*slow"
<<<"$out" grepQuietInverse -E "^error:.*depends-on-slow"
<<<"$out" grepQuietInverse -E "^error:.*depends-on-fail"
# Error messages should not be empty (end with just "failed:")
<<<"$out" grepQuietInverse -E "^error:.*failed: *$"
fi

View File

@@ -6,7 +6,7 @@ export NIX_TESTS_CA_BY_DEFAULT=1
drvPath=$(nix-instantiate ../simple.nix)
nix derivation show "$drvPath" | jq '.derivations[]' > "$TEST_HOME"/simple.json
nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json
drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json)
@@ -27,5 +27,5 @@ drvPath4=$(nix derivation add < "$TEST_HOME"/foo.json)
[[ -e "$drvPath3" ]]
# The modified derivation read back as JSON matches
nix derivation show "$drvPath3" | jq '.derivations[]' > "$TEST_HOME"/foo-read.json
nix derivation show "$drvPath3" | jq .[] > "$TEST_HOME"/foo-read.json
diff "$TEST_HOME"/foo.json "$TEST_HOME"/foo-read.json

View File

@@ -1,64 +0,0 @@
# Regression test for cancelled builds not being reported as failures.
#
# Scenario: When a build fails while other builds are running, those other
# builds (and their dependents) get cancelled. Previously, cancelled builds
# were incorrectly reported as failures with empty error messages.
#
# Uses a fifo for synchronization: fast-fail waits for slow to start before
# failing, ensuring slow is actually running when it gets cancelled.
#
# See: tests/functional/build.sh (search for "cancelled-builds")
{
outputs =
{ self }:
let
config = import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix";
in
with config;
{
checks.${system} = {
# A derivation that signals it started via fifo, then waits
slow = mkDerivation {
name = "slow";
buildCommand = ''
echo "slow: started, signaling via fifo"
echo started > /cancelled-builds-fifo/fifo
echo "slow: waiting..."
sleep 10
touch $out
'';
};
# Depends on slow - will be cancelled when fast-fail fails
depends-on-slow = mkDerivation {
name = "depends-on-slow";
slow = self.checks.${system}.slow;
buildCommand = ''
echo "depends-on-slow: slow finished at $slow"
touch $out
'';
};
# Waits for slow to start via fifo, then fails
fast-fail = mkDerivation {
name = "fast-fail";
buildCommand = ''
echo "fast-fail: waiting for slow to start..."
read line < /cancelled-builds-fifo/fifo
echo "fast-fail: slow started, now failing" >&2
exit 1
'';
};
# Depends on fast-fail - will fail with DependencyFailed
depends-on-fail = mkDerivation {
name = "depends-on-fail";
fast-fail = self.checks.${system}.fast-fail;
buildCommand = ''
echo "depends-on-fail: fast-fail finished (should never get here)"
touch $out
'';
};
};
};
}

View File

@@ -4,7 +4,7 @@ source common.sh
drvPath=$(nix-instantiate simple.nix)
nix derivation show "$drvPath" | jq '.derivations[]' > "$TEST_HOME/simple.json"
nix derivation show "$drvPath" | jq '.[]' > "$TEST_HOME/simple.json"
# Round tripping to JSON works
drvPath2=$(nix derivation add < "$TEST_HOME/simple.json")

View File

@@ -10,36 +10,3 @@ if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not
expectStderr "$expected" nix-build ./text-hashed-output.nix -A failingWrapper --no-out-link \
| grepQuiet "build of resolved derivation '.*use-dynamic-drv-in-non-dynamic-drv-wrong.drv' failed"
# Test that error messages are not empty when a producer derivation fails.
# This exercises the nrFailed path in DerivationTrampolineGoal::init().
#
# Using `nix build --expr` with builtins.outputOf creates a top-level
# DerivationTrampolineGoal that goes through buildPathsWithResults.
# When the producer fails, the nrFailed path must use doneFailure (not amDone)
# to set buildResult.inner with a proper error message.
#
# Without the fix, the error message would be empty because amDone doesn't
# set buildResult.inner, so rethrow() throws Error("") - an empty message.
out=$(nix build --impure --no-link --expr '
let
config = import (builtins.getEnv "_NIX_TEST_BUILD_DIR" + "/config.nix");
inherit (config) mkDerivation;
# A CA derivation that fails before producing a .drv
failingProducer = mkDerivation {
name = "failing-producer";
buildCommand = "echo This producer fails; exit 1";
__contentAddressed = true;
outputHashMode = "text";
outputHashAlgo = "sha256";
};
in
# Build the dynamic derivation output directly - this creates a top-level
# DerivationTrampolineGoal, not a nested one inside a DerivationGoal
builtins.outputOf failingProducer.outPath "out"
' 2>&1) || true
# The error message must NOT be empty - it should mention the failed derivation
echo "$out" | grepQuiet "failed to obtain derivation of"

View File

@@ -18,7 +18,7 @@ mkDerivation rec {
PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH
# JSON of pre-existing drv
nix derivation show $drv | jq '.derivations[]' > drv0.json
nix derivation show $drv | jq .[] > drv0.json
# Fix name
jq < drv0.json '.name = "${innerName}"' > drv1.json

View File

@@ -16,7 +16,7 @@ printf 0 > "$TEST_ROOT"/counter
# `nix derivation add` with impure derivations work
drvPath=$(nix-instantiate ./impure-derivations.nix -A impure)
nix derivation show "$drvPath" | jq '.derivations[]' > "$TEST_HOME"/impure-drv.json
nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/impure-drv.json
drvPath2=$(nix derivation add < "$TEST_HOME"/impure-drv.json)
[[ "$drvPath" = "$drvPath2" ]]
@@ -50,8 +50,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm
(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
[[ $(nix derivation show "$drvPath" | jq ".derivations[\"$(basename "$drvPath")\"].outputs.out.impure") = true ]]
[[ $(nix derivation show "$drvPath" | jq ".derivations[\"$(basename "$drvPath")\"].outputs.stuff.impure") = true ]]
[[ $(nix derivation show "$drvPath" | jq ".[\"$(basename "$drvPath")\"].outputs.out.impure") = true ]]
[[ $(nix derivation show "$drvPath" | jq ".[\"$(basename "$drvPath")\"].outputs.stuff.impure") = true ]]
# Fixed-output derivations *can* depend on impure derivations.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)

View File

@@ -1,16 +0,0 @@
error:
… while evaluating the attribute 'absolutePath'
at /pwd/lang/eval-fail-readDir-nonexistent-1.nix:2:3:
1| {
2| absolutePath = builtins.readDir /this/path/really/should/not/exist;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-nonexistent-1.nix:2:18:
1| {
2| absolutePath = builtins.readDir /this/path/really/should/not/exist;
| ^
3| }
error: path '/this/path/really/should/not/exist' does not exist

View File

@@ -1,3 +0,0 @@
{
absolutePath = builtins.readDir /this/path/really/should/not/exist;
}

View File

@@ -1,16 +0,0 @@
error:
… while evaluating the attribute 'relativePath'
at /pwd/lang/eval-fail-readDir-nonexistent-2.nix:2:3:
1| {
2| relativePath = builtins.readDir ./this/path/really/should/not/exist;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-nonexistent-2.nix:2:18:
1| {
2| relativePath = builtins.readDir ./this/path/really/should/not/exist;
| ^
3| }
error: path '/pwd/lang/this/path/really/should/not/exist' does not exist

View File

@@ -1,3 +0,0 @@
{
relativePath = builtins.readDir ./this/path/really/should/not/exist;
}

View File

@@ -1,16 +0,0 @@
error:
… while evaluating the attribute 'regularFile'
at /pwd/lang/eval-fail-readDir-not-a-directory-1.nix:2:3:
1| {
2| regularFile = builtins.readDir ./readDir/bar;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-not-a-directory-1.nix:2:17:
1| {
2| regularFile = builtins.readDir ./readDir/bar;
| ^
3| }
error: cannot read directory "/pwd/lang/readDir/bar": Not a directory

View File

@@ -1,3 +0,0 @@
{
regularFile = builtins.readDir ./readDir/bar;
}

View File

@@ -1,16 +0,0 @@
error:
… while evaluating the attribute 'symlinkedRegularFile'
at /pwd/lang/eval-fail-readDir-not-a-directory-2.nix:2:3:
1| {
2| symlinkedRegularFile = builtins.readDir ./readDir/linked;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-not-a-directory-2.nix:2:26:
1| {
2| symlinkedRegularFile = builtins.readDir ./readDir/linked;
| ^
3| }
error: cannot read directory "/pwd/lang/readDir/foo/git-hates-directories": Not a directory

View File

@@ -1,3 +0,0 @@
{
symlinkedRegularFile = builtins.readDir ./readDir/linked;
}

View File

@@ -1 +0,0 @@
{ git-hates-directories = "regular"; }

View File

@@ -1 +0,0 @@
builtins.readDir ./readDir/ldir

View File

@@ -49,4 +49,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "
# Check it works with the expected structured attrs
hacky=$(nix-instantiate --expr "$hackyExpr")
nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'
nix derivation show "$hacky" | jq --exit-status '."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'

View File

@@ -147,7 +147,7 @@ in
else:
machine.fail(f"nix path-info {pkg}")
def setup_s3(populate_bucket=[], public=False, versioned=False, profiles=None):
def setup_s3(populate_bucket=[], public=False, versioned=False):
"""
Decorator that creates/destroys a unique bucket for each test.
Optionally pre-populates bucket with specified packages.
@@ -157,22 +157,9 @@ in
populate_bucket: List of packages to upload before test runs
public: If True, make the bucket publicly accessible
versioned: If True, enable versioning on the bucket before populating
profiles: Dict of AWS profiles to create, e.g.:
{"valid": {"access_key": "...", "secret_key": "..."},
"invalid": {"access_key": "WRONG", "secret_key": "WRONG"}}
Profiles are created on the client machine at /root/.aws/credentials
"""
def decorator(test_func):
def wrapper():
# Restart nix-daemon on both machines to clear the credential provider cache.
# The AwsCredentialProviderImpl singleton persists in the daemon process,
# and its cache can cause credentials from previous tests to be reused.
# We reset-failed first to avoid systemd's start rate limiting.
server.succeed("systemctl reset-failed nix-daemon.service nix-daemon.socket")
server.succeed("systemctl restart nix-daemon")
client.succeed("systemctl reset-failed nix-daemon.service nix-daemon.socket")
client.succeed("systemctl restart nix-daemon")
bucket = str(uuid.uuid4())
server.succeed(f"mc mb minio/{bucket}")
try:
@@ -180,15 +167,6 @@ in
server.succeed(f"mc anonymous set download minio/{bucket}")
if versioned:
server.succeed(f"mc version enable minio/{bucket}")
if profiles:
# Build credentials file content
creds_content = ""
for name, creds in profiles.items():
creds_content += f"[{name}]\n"
creds_content += f"aws_access_key_id = {creds['access_key']}\n"
creds_content += f"aws_secret_access_key = {creds['secret_key']}\n\n"
client.succeed("mkdir -p /root/.aws")
client.succeed(f"cat > /root/.aws/credentials << 'AWSCREDS'\n{creds_content}AWSCREDS")
if populate_bucket:
store_url = make_s3_url(bucket)
for pkg in populate_bucket:
@@ -196,9 +174,6 @@ in
test_func(bucket)
finally:
server.succeed(f"mc rb --force minio/{bucket}")
# Clean up AWS profiles if created
if profiles:
client.succeed("rm -rf /root/.aws")
# Clean up client store - only delete if path exists
for pkg in PKGS.values():
client.succeed(f"[ ! -e {pkg} ] || nix store delete --ignore-liveness {pkg}")
@@ -227,40 +202,7 @@ in
"Credential provider caching failed"
)
@setup_s3()
def test_aws_log_integration(bucket):
"""Test that AWS SDK logs are properly routed through Nix logger"""
print("\n=== Testing AWS Log Integration ===")
store_url = make_s3_url(bucket)
# With default verbosity, AWS noise should NOT appear
# All AWS messages are demoted to lvlDebug or lvlVomit
output_default = server.succeed(
f"{ENV_WITH_CREDS} nix copy --to '{store_url}' {PKGS['A']} 2>&1"
)
if "(aws:" in output_default:
print("Output at default verbosity:")
print(output_default)
raise Exception("Found AWS noise at default verbosity")
# With --debug (lvlDebug), we should see AWS messages with (aws:subject) prefix
output_debug = server.succeed(
f"{ENV_WITH_CREDS} nix copy --debug --to '{store_url}' {PKGS['B']} 2>&1"
)
# Check for the (aws:subject) prefix format
if "(aws:" not in output_debug:
print("Output at --debug verbosity:")
print(output_debug)
raise Exception("Expected to see (aws:subject) prefix in debug output")
# Should also see Nix's own credential provider creation message
if "creating new AWS credential provider" not in output_debug:
print("Debug output:")
print(output_debug)
raise Exception("Expected to see credential provider creation at debug level")
print(" Credential provider created once and cached")
@setup_s3(populate_bucket=[PKGS['A']])
def test_fetchurl_basic(bucket):
@@ -276,6 +218,8 @@ in
f"'builtins.fetchurl {{ name = \"foo\"; url = \"{cache_info_url}\"; }}'"
)
print(" builtins.fetchurl works with s3:// URLs")
@setup_s3()
def test_error_message_formatting(bucket):
"""Verify error messages display URLs correctly"""
@@ -294,6 +238,8 @@ in
print(error_msg)
raise Exception("Error message formatting failed - should show actual URL, not %s placeholder")
print(" Error messages format URLs correctly")
@setup_s3(populate_bucket=[PKGS['A']])
def test_fork_credential_preresolution(bucket):
"""Test credential pre-resolution in forked processes"""
@@ -333,6 +279,8 @@ in
print(output)
raise Exception("Expected to find FileTransfer creation in forked process")
print(" Forked process creates fresh FileTransfer")
# Verify pre-resolution in parent
required_messages = [
"Pre-resolving AWS credentials for S3 URL in builtin:fetchurl",
@@ -345,6 +293,8 @@ in
print(output)
raise Exception(f"Missing expected message: {msg}")
print(" Parent pre-resolves credentials")
# Verify child uses pre-resolved credentials
if "Using pre-resolved AWS credentials from parent process" not in output:
print("Debug output:")
@@ -368,6 +318,8 @@ in
print(output)
raise Exception(f"Child process (pid={child_pid}) should NOT create new credential providers")
print(" Child uses pre-resolved credentials (no new providers)")
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B'], PKGS['C']])
def test_store_operations(bucket):
"""Test nix store info and copy operations"""
@@ -385,6 +337,8 @@ in
if not store_info.get("url"):
raise Exception("Store should have a URL")
print(f" Store URL: {store_info['url']}")
# Test copy from store
verify_packages_in_store(client, PKGS['A'], should_exist=False)
@@ -402,6 +356,9 @@ in
verify_packages_in_store(client, [PKGS['A'], PKGS['B'], PKGS['C']])
print(" nix copy works")
print(" Credentials cached on client")
@setup_s3(populate_bucket=[PKGS['A'], PKGS['B']], public=True)
def test_public_bucket_operations(bucket):
"""Test store operations on public bucket without credentials"""
@@ -411,6 +368,7 @@ in
# Verify store info works without credentials
client.succeed(f"nix store info --store '{store_url}' >&2")
print(" nix store info works without credentials")
# Get and validate store info JSON
info_json = client.succeed(f"nix store info --json --store '{store_url}'")
@@ -419,6 +377,8 @@ in
if not store_info.get("url"):
raise Exception("Store should have a URL")
print(f" Store URL: {store_info['url']}")
# Verify packages are not yet in client store
verify_packages_in_store(client, [PKGS['A'], PKGS['B']], should_exist=False)
@@ -431,6 +391,8 @@ in
# Verify packages were copied successfully
verify_packages_in_store(client, [PKGS['A'], PKGS['B']])
print(" nix copy from public bucket works without credentials")
@setup_s3(populate_bucket=[PKGS['A']])
def test_url_format_variations(bucket):
"""Test different S3 URL parameter combinations"""
@@ -439,10 +401,12 @@ in
# Test parameter order variation (region before endpoint)
url1 = f"s3://{bucket}?region={REGION}&endpoint={ENDPOINT}"
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url1}' >&2")
print(" Parameter order: region before endpoint works")
# Test parameter order variation (endpoint before region)
url2 = f"s3://{bucket}?endpoint={ENDPOINT}&region={REGION}"
client.succeed(f"{ENV_WITH_CREDS} nix store info --store '{url2}' >&2")
print(" Parameter order: endpoint before region works")
@setup_s3(populate_bucket=[PKGS['A']])
def test_concurrent_fetches(bucket):
@@ -496,6 +460,9 @@ in
providers_created = output.count("creating new AWS credential provider")
transfers_created = output.count("builtin:fetchurl creating fresh FileTransfer instance")
print(f" {providers_created} credential providers created")
print(f" {transfers_created} FileTransfer instances created")
if transfers_created != 5:
print("Debug output:")
print(output)
@@ -521,10 +488,14 @@ in
pkg_hash = get_package_hash(PKGS['B'])
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "gzip")
print(" .narinfo has Content-Encoding: gzip")
# Verify client can download and decompress
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['B']}")
verify_packages_in_store(client, PKGS['B'])
print(" Client decompressed .narinfo successfully")
@setup_s3()
def test_compression_mixed(bucket):
"""Test mixed compression (narinfo=xz, ls=gzip)"""
@@ -541,14 +512,18 @@ in
# Verify .narinfo has xz compression
verify_content_encoding(server, bucket, f"{pkg_hash}.narinfo", "xz")
print(" .narinfo has Content-Encoding: xz")
# Verify .ls has gzip compression
verify_content_encoding(server, bucket, f"{pkg_hash}.ls", "gzip")
print(" .ls has Content-Encoding: gzip")
# Verify client can download with mixed compression
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' --no-check-sigs {PKGS['C']}")
verify_packages_in_store(client, PKGS['C'])
print(" Client downloaded package with mixed compression")
@setup_s3()
def test_compression_disabled(bucket):
"""Verify no compression by default"""
@@ -560,6 +535,8 @@ in
pkg_hash = get_package_hash(PKGS['A'])
verify_no_compression(server, bucket, f"{pkg_hash}.narinfo")
print(" No compression applied by default")
@setup_s3()
def test_nix_prefetch_url(bucket):
"""Test that nix-prefetch-url retrieves actual file content from S3, not empty files (issue #8862)"""
@@ -579,6 +556,8 @@ in
"nix hash file --type sha256 --base32 /tmp/test-file.txt"
).strip()
print(f" Uploaded test file to S3 ({test_file_size} bytes)")
# Use nix-prefetch-url to download from S3
s3_url = make_s3_url(bucket, path="/test-file.txt")
@@ -598,6 +577,8 @@ in
f"Hash mismatch: expected {expected_hash}, got {prefetch_hash}"
)
print(" nix-prefetch-url completed with correct hash")
# Verify the downloaded file is NOT empty (the bug in #8862)
file_size = int(client.succeed(f"stat -c %s {store_path}").strip())
@@ -609,12 +590,16 @@ in
f"File size mismatch: expected {test_file_size}, got {file_size}"
)
print(f" File has correct size ({file_size} bytes, not empty)")
# Verify actual content matches by comparing hashes instead of printing entire file
downloaded_hash = client.succeed(f"nix hash file --type sha256 --base32 {store_path}").strip()
if downloaded_hash != expected_hash:
raise Exception(f"Content hash mismatch: expected {expected_hash}, got {downloaded_hash}")
print(" File content verified correct (hash matches)")
@setup_s3(populate_bucket=[PKGS['A']], versioned=True)
def test_versioned_urls(bucket):
"""Test that versionId parameter is accepted in S3 URLs"""
@@ -628,6 +613,7 @@ in
f"{ENV_WITH_CREDS} nix eval --impure --expr "
f"'builtins.fetchurl {{ name = \"cache-info\"; url = \"{cache_info_url}\"; }}'"
)
print(" Fetch without versionId works")
# List versions to get a version ID
# MinIO output format: [timestamp] size tier versionId versionNumber method filename
@@ -641,6 +627,8 @@ in
raise Exception("Could not extract version ID from MinIO output")
version_id = version_match.group(1)
print(f" Found version ID: {version_id}")
# Version ID should not be "null" since versioning was enabled before upload
if version_id == "null":
raise Exception("Version ID is 'null' - versioning may not be working correctly")
@@ -651,6 +639,7 @@ in
f"{ENV_WITH_CREDS} nix eval --impure --expr "
f"'builtins.fetchurl {{ name = \"cache-info-versioned\"; url = \"{versioned_url}\"; }}'"
)
print(" Fetch with versionId parameter works")
@setup_s3()
def test_multipart_upload_basic(bucket):
@@ -686,9 +675,13 @@ in
print(output)
raise Exception(f"Expected '{expected_msg}' in output")
print(f" Multipart upload used with {expected_parts} parts")
client.succeed(f"{ENV_WITH_CREDS} nix copy --from '{store_url}' {large_pkg} --no-check-sigs")
verify_packages_in_store(client, large_pkg, should_exist=True)
print(" Large file downloaded and verified")
@setup_s3()
def test_multipart_threshold(bucket):
"""Test that files below threshold use regular upload"""
@@ -711,9 +704,13 @@ in
if "using S3 regular upload" not in output:
raise Exception("Expected regular upload to be used")
print(" Regular upload used for file below threshold")
client.succeed(f"{ENV_WITH_CREDS} nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}")
verify_packages_in_store(client, PKGS['A'], should_exist=True)
print(" Small file uploaded and verified")
@setup_s3()
def test_multipart_with_log_compression(bucket):
"""Test multipart upload with compressed build logs"""
@@ -765,99 +762,7 @@ in
print(output)
raise Exception("Expected multipart completion message")
@setup_s3(
populate_bucket=[PKGS['A']],
profiles={
"valid": {"access_key": ACCESS_KEY, "secret_key": SECRET_KEY},
"invalid": {"access_key": "INVALIDKEY", "secret_key": "INVALIDSECRET"},
}
)
def test_profile_credentials(bucket):
"""Test that profile-based credentials work without environment variables"""
print("\n=== Testing Profile-Based Credentials ===")
store_url = make_s3_url(bucket, profile="valid")
# Verify store info works with profile credentials (no env vars)
client.succeed(f"HOME=/root nix store info --store '{store_url}' >&2")
# Verify we can copy from the store using profile
verify_packages_in_store(client, PKGS['A'], should_exist=False)
client.succeed(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']}")
verify_packages_in_store(client, PKGS['A'])
# Clean up the package we just copied so we can test invalid profile
client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}")
verify_packages_in_store(client, PKGS['A'], should_exist=False)
# Verify invalid profile fails when trying to copy
invalid_url = make_s3_url(bucket, profile="invalid")
client.fail(f"HOME=/root nix copy --no-check-sigs --from '{invalid_url}' {PKGS['A']} 2>&1")
@setup_s3(
populate_bucket=[PKGS['A']],
profiles={
"wrong": {"access_key": "WRONGKEY", "secret_key": "WRONGSECRET"},
}
)
def test_env_vars_precedence(bucket):
"""Test that environment variables take precedence over profile credentials"""
print("\n=== Testing Environment Variables Precedence ===")
# Use profile with wrong credentials, but provide correct creds via env vars
store_url = make_s3_url(bucket, profile="wrong")
# Ensure package is not in client store
verify_packages_in_store(client, PKGS['A'], should_exist=False)
# This should succeed because env vars (correct) override profile (wrong)
output = client.succeed(
f"HOME=/root {ENV_WITH_CREDS} nix copy --no-check-sigs --debug --from '{store_url}' {PKGS['A']} 2>&1"
)
# Verify the credential chain shows Environment provider was added
if "Added AWS Environment Credential Provider" not in output:
print("Debug output:")
print(output)
raise Exception("Expected Environment provider to be added to chain")
# Clean up the package so we can test again without env vars
client.succeed(f"nix store delete --ignore-liveness {PKGS['A']}")
verify_packages_in_store(client, PKGS['A'], should_exist=False)
# Without env vars, same URL should fail (proving profile creds are actually wrong)
client.fail(f"HOME=/root nix copy --no-check-sigs --from '{store_url}' {PKGS['A']} 2>&1")
@setup_s3(
populate_bucket=[PKGS['A']],
profiles={
"testprofile": {"access_key": ACCESS_KEY, "secret_key": SECRET_KEY},
}
)
def test_credential_provider_chain(bucket):
"""Test that debug logging shows which providers are added to the chain"""
print("\n=== Testing Credential Provider Chain Logging ===")
store_url = make_s3_url(bucket, profile="testprofile")
output = client.succeed(
f"HOME=/root nix store info --debug --store '{store_url}' 2>&1"
)
# For a named profile, we expect to see these providers in the chain
expected_providers = ["Environment", "Profile", "IMDS"]
for provider in expected_providers:
msg = f"Added AWS {provider} Credential Provider to chain for profile 'testprofile'"
if msg not in output:
print("Debug output:")
print(output)
raise Exception(f"Expected to find: {msg}")
# SSO should be skipped (no SSO config for this profile)
if "Skipped AWS SSO Credential Provider for profile 'testprofile'" not in output:
print("Debug output:")
print(output)
raise Exception("Expected SSO provider to be skipped")
print(" Compressed log uploaded with multipart")
# ============================================================================
# Main Test Execution
@@ -877,7 +782,6 @@ in
# Run tests (each gets isolated bucket via decorator)
test_credential_caching()
test_aws_log_integration()
test_fetchurl_basic()
test_error_message_formatting()
test_fork_credential_preresolution()
@@ -893,8 +797,9 @@ in
test_multipart_upload_basic()
test_multipart_threshold()
test_multipart_with_log_compression()
test_profile_credentials()
test_env_vars_precedence()
test_credential_provider_chain()
print("\n" + "="*80)
print(" All S3 Binary Cache Store Tests Passed!")
print("="*80)
'';
}