Compare commits
93
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1c0f416cb | ||
|
|
8104858643 | ||
|
|
a6c1d5637a | ||
|
|
27006cc8a9 | ||
|
|
06f21596a0 | ||
|
|
9254fab407 | ||
|
|
994324feda | ||
|
|
1f739961e5 | ||
|
|
cccfa385e6 | ||
|
|
9d2100a165 | ||
|
|
4769f3c0b2 | ||
|
|
ab354dc8f6 | ||
|
|
188cb798ad | ||
|
|
1aa7ab0dcf | ||
|
|
208ed3c538 | ||
|
|
b6add8dcc6 | ||
|
|
79750a3ccc | ||
|
|
30cd9e43e1 | ||
|
|
0695630eb5 | ||
|
|
89dc57f6aa | ||
|
|
f274a7273a | ||
|
|
675656ffba | ||
|
|
a5edc2d921 | ||
|
|
2f092870e4 | ||
|
|
b39da9c0c2 | ||
|
|
f536b25367 | ||
|
|
017fae3f14 | ||
|
|
018d6462de | ||
|
|
4a5d960952 | ||
|
|
8cf8a9151a | ||
|
|
4060ec3a8c | ||
|
|
e0830681e2 | ||
|
|
9f2795e588 | ||
|
|
12cee327a0 | ||
|
|
3b73dcba39 | ||
|
|
dfad4b1403 | ||
|
|
5f69fd3e8d | ||
|
|
47416968d2 | ||
|
|
ce38abb697 | ||
|
|
a38fc659cc | ||
|
|
85bbfd4493 | ||
|
|
d5d7594029 | ||
|
|
1fc5648204 | ||
|
|
d7e0bcaa51 | ||
|
|
4227d24bc3 | ||
|
|
7720dad11f | ||
|
|
832b81761e | ||
|
|
1c63cf4001 | ||
|
|
df7542247e | ||
|
|
49f666c64d | ||
|
|
11f5a3124b | ||
|
|
92e698426b | ||
|
|
906334686c | ||
|
|
0ffe83aa14 | ||
|
|
8e044f1ed0 | ||
|
|
453dbab1e8 | ||
|
|
fc81840a8e | ||
|
|
71bdb33a36 | ||
|
|
0595c5f7ee | ||
|
|
11f108d898 | ||
|
|
128b2b5c56 | ||
|
|
508d4463e5 | ||
|
|
3c8e45c061 | ||
|
|
ec91479076 | ||
|
|
b398c14045 | ||
|
|
9a6f1e6266 | ||
|
|
1c728ce0de | ||
|
|
e145632aef | ||
|
|
5cdf2a19bd | ||
|
|
bb74677b08 | ||
|
|
3cfac9b079 | ||
|
|
198628790b | ||
|
|
54d2268d84 | ||
|
|
8c74aadbf7 | ||
|
|
3a62be7227 | ||
|
|
a6eb2e91b7 | ||
|
|
76c09bf3d4 | ||
|
|
de6fdb7da5 | ||
|
|
b54dfb66dd | ||
|
|
bb718d20a2 | ||
|
|
3b3bd018a5 | ||
|
|
26b86a02db | ||
|
|
8358409fd0 | ||
|
|
46670a7f46 | ||
|
|
ea96e6d07c | ||
|
|
7e3de5361a | ||
|
|
7b3d7eb634 | ||
|
|
819a61acae | ||
|
|
ccba158780 | ||
|
|
4945c38b88 | ||
|
|
0f18076f3a | ||
|
|
c6ddc5cf1d | ||
|
|
af6326dfa4 |
No files matched your search
@@ -24,8 +24,8 @@ inputs:
|
||||
description: "Github token"
|
||||
required: true
|
||||
use_cache:
|
||||
description: "Whether to setup github actions cache (not implemented currently)"
|
||||
default: false
|
||||
description: "Whether to setup magic-nix-cache"
|
||||
default: true
|
||||
required: false
|
||||
runs:
|
||||
using: "composite"
|
||||
@@ -122,3 +122,10 @@ runs:
|
||||
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
|
||||
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
|
||||
extra-conf: ${{ inputs.extra_nix_config }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
|
||||
if: ${{ inputs.use_cache == 'true' }}
|
||||
with:
|
||||
diagnostic-endpoint: ''
|
||||
use-flakehub: false
|
||||
use-gha-cache: true
|
||||
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
|
||||
+79
-28
@@ -125,13 +125,13 @@ jobs:
|
||||
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
|
||||
if: ${{ matrix.instrumented }}
|
||||
- name: Upload coverage reports
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: coverage-reports
|
||||
path: coverage-reports/
|
||||
if: ${{ matrix.instrumented }}
|
||||
- name: Upload installer tarball
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: installer-${{matrix.os}}
|
||||
path: out/*
|
||||
@@ -164,7 +164,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Download installer tarball
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: installer-${{matrix.os}}
|
||||
path: out
|
||||
@@ -174,7 +174,7 @@ jobs:
|
||||
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
||||
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
|
||||
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
|
||||
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
|
||||
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
|
||||
if: ${{ !matrix.experimental-installer }}
|
||||
with:
|
||||
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
@@ -197,20 +197,79 @@ jobs:
|
||||
- run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
|
||||
- run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
|
||||
|
||||
# Steps to test CI automation in your own fork.
|
||||
# 1. Sign-up for https://hub.docker.com/
|
||||
# 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
|
||||
# 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
|
||||
check_secrets:
|
||||
permissions:
|
||||
contents: none
|
||||
name: Check presence of secrets
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
docker: ${{ steps.secret.outputs.docker }}
|
||||
steps:
|
||||
- name: Check for DockerHub secrets
|
||||
id: secret
|
||||
env:
|
||||
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
|
||||
|
||||
docker_push_image:
|
||||
name: Push docker image to DockerHub and GHCR
|
||||
needs: [flake_regressions, installer_test]
|
||||
if: github.event_name == 'push' && github.ref_name == 'master'
|
||||
uses: ./.github/workflows/docker-push.yml
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
is_master: true
|
||||
needs: [tests, check_secrets]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
if: >-
|
||||
needs.check_secrets.outputs.docker == 'true' &&
|
||||
github.event_name == 'push' &&
|
||||
github.ref_name == 'master'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
dogfood: false
|
||||
extra_nix_config: |
|
||||
experimental-features = flakes nix-command
|
||||
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
|
||||
- run: nix build .#dockerImage -L
|
||||
- run: docker load -i ./result/image.tar.gz
|
||||
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
|
||||
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
|
||||
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
|
||||
#
|
||||
# Push to Docker Hub first
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
|
||||
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
|
||||
# Push to GitHub Container Registry as well
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Push image
|
||||
run: |
|
||||
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
|
||||
# Change all uppercase to lowercase
|
||||
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
|
||||
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:latest
|
||||
docker push $IMAGE_ID:$NIX_VERSION
|
||||
docker push $IMAGE_ID:latest
|
||||
# deprecated 2024-02-24
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:master
|
||||
docker push $IMAGE_ID:master
|
||||
|
||||
flake_regressions:
|
||||
needs: tests
|
||||
@@ -228,21 +287,13 @@ jobs:
|
||||
with:
|
||||
repository: NixOS/flake-regressions-data
|
||||
path: flake-regressions/tests
|
||||
- name: Download installer tarball
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
name: installer-linux
|
||||
path: out
|
||||
- name: Looking up the installer tarball URL
|
||||
id: installer-tarball-url
|
||||
run: |
|
||||
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
||||
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
|
||||
with:
|
||||
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
- name: Run flake regressions tests
|
||||
run: MAX_FLAKES=25 flake-regressions/eval-all.sh
|
||||
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
|
||||
extra_nix_config:
|
||||
experimental-features = nix-command flakes
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
|
||||
|
||||
profile_build:
|
||||
needs: tests
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
name: "Push Docker Image"
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: "Git ref to build the docker image from"
|
||||
required: true
|
||||
type: string
|
||||
is_master:
|
||||
description: "Whether run from master branch"
|
||||
required: true
|
||||
type: boolean
|
||||
secrets:
|
||||
DOCKERHUB_USERNAME:
|
||||
required: true
|
||||
DOCKERHUB_TOKEN:
|
||||
required: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
# Steps to test CI automation in your own fork.
|
||||
# 1. Sign-up for https://hub.docker.com/
|
||||
# 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
|
||||
# 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
|
||||
check_secrets:
|
||||
permissions:
|
||||
contents: none
|
||||
name: Check presence of secrets
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
docker: ${{ steps.secret.outputs.docker }}
|
||||
steps:
|
||||
- name: Check for DockerHub secrets
|
||||
id: secret
|
||||
env:
|
||||
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
|
||||
|
||||
push:
|
||||
name: Push docker image to DockerHub and GHCR
|
||||
needs: [check_secrets]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
if: needs.check_secrets.outputs.docker == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.ref }}
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
dogfood: false
|
||||
extra_nix_config: |
|
||||
experimental-features = flakes nix-command
|
||||
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
|
||||
- run: nix build .#dockerImage -L
|
||||
- run: docker load -i ./result/image.tar.gz
|
||||
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
|
||||
#
|
||||
# Push to Docker Hub first
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Push to Docker Hub
|
||||
env:
|
||||
IS_MASTER: ${{ inputs.is_master }}
|
||||
DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_USERNAME }}/nix
|
||||
run: |
|
||||
docker tag nix:$NIX_VERSION $DOCKERHUB_REPO:$NIX_VERSION
|
||||
docker push $DOCKERHUB_REPO:$NIX_VERSION
|
||||
if [ "$IS_MASTER" = "true" ]; then
|
||||
docker tag nix:$NIX_VERSION $DOCKERHUB_REPO:master
|
||||
docker push $DOCKERHUB_REPO:master
|
||||
fi
|
||||
# Push to GitHub Container Registry as well
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Push to GHCR
|
||||
env:
|
||||
IS_MASTER: ${{ inputs.is_master }}
|
||||
run: |
|
||||
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
|
||||
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
|
||||
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
|
||||
docker push $IMAGE_ID:$NIX_VERSION
|
||||
if [ "$IS_MASTER" = "true" ]; then
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:master
|
||||
docker push $IMAGE_ID:master
|
||||
fi
|
||||
@@ -1,80 +0,0 @@
|
||||
name: Upload Release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
eval_id:
|
||||
description: "Hydra evaluation ID"
|
||||
required: true
|
||||
type: number
|
||||
is_latest:
|
||||
description: "Mark as latest release"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-24.04
|
||||
environment: releases
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
dogfood: false # Use stable version
|
||||
use_cache: false # Don't want any cache injection shenanigans
|
||||
extra_nix_config: |
|
||||
experimental-features = nix-command flakes
|
||||
- name: Set NIX_PATH from flake input
|
||||
run: |
|
||||
NIXPKGS_PATH=$(nix build --inputs-from .# nixpkgs#path --print-out-paths --no-link)
|
||||
# Shebangs with perl have issues. Pin nixpkgs this way. nix shell should maybe
|
||||
# get the same uberhack that nix-shell has to support it.
|
||||
echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV"
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
|
||||
with:
|
||||
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
|
||||
role-session-name: nix-release-oidc-${{ github.run_id }}
|
||||
aws-region: eu-west-1
|
||||
- name: Disable containerd image store
|
||||
run: |
|
||||
# Docker 28+ defaults to the containerd image store, which
|
||||
# pushes layers uncompressed instead of gzip. OCI clients
|
||||
# that only support gzip (e.g. go-containerregistry) fail
|
||||
# with "gzip: invalid header". Disabling the containerd
|
||||
# snapshotter restores the classic storage driver, which
|
||||
# preserves gzip-compressed layers through the
|
||||
# `docker load` / `docker push` pipeline.
|
||||
echo '{"features":{"containerd-snapshotter":false}}' | sudo tee /etc/docker/daemon.json > /dev/null
|
||||
sudo systemctl restart docker
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Upload release
|
||||
run: |
|
||||
./maintainers/upload-release.pl \
|
||||
${{ inputs.eval_id }} \
|
||||
--skip-git
|
||||
env:
|
||||
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
|
||||
- name: Push to GHCR
|
||||
run: |
|
||||
DOCKER_OWNER="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[A-Z]' '[a-z]')/nix"
|
||||
./maintainers/upload-release.pl \
|
||||
${{ inputs.eval_id }} \
|
||||
--skip-git \
|
||||
--skip-s3 \
|
||||
--docker-owner "$DOCKER_OWNER"
|
||||
env:
|
||||
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
|
||||
@@ -26,7 +26,6 @@ bash = find_program('bash', native : true)
|
||||
# HTML manual dependencies (conditional)
|
||||
if get_option('html-manual')
|
||||
mdbook = find_program('mdbook', native : true)
|
||||
rsync = find_program('rsync', required : true, native : true)
|
||||
endif
|
||||
|
||||
pymod = import('python')
|
||||
@@ -126,7 +125,12 @@ if get_option('html-manual')
|
||||
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
|
||||
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
|
||||
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
|
||||
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
|
||||
# Copy source to build directory, excluding the build directory itself
|
||||
# (which is present when built as an individual component).
|
||||
# Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests).
|
||||
(cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -)
|
||||
chmod -R u+w @2@
|
||||
find @2@ -name '*.drv' -delete
|
||||
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
|
||||
rm -rf @2@/manual
|
||||
mv @2@/html @2@/manual
|
||||
@@ -138,7 +142,6 @@ if get_option('html-manual')
|
||||
mdbook.full_path(),
|
||||
meson.current_build_dir(),
|
||||
meson.project_version(),
|
||||
rsync.full_path(),
|
||||
),
|
||||
],
|
||||
input : [
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
mdbook,
|
||||
jq,
|
||||
python3,
|
||||
rsync,
|
||||
nix-cli,
|
||||
changelog-d,
|
||||
json-schema-for-humans,
|
||||
@@ -54,6 +53,8 @@ mkMesonDerivation (finalAttrs: {
|
||||
../../src/libstore-tests/data/nar-info
|
||||
../../src/libstore-tests/data/build-result
|
||||
../../src/libstore-tests/data/dummy-store
|
||||
# For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/
|
||||
../../tests/functional/derivation
|
||||
# Too many different types of files to filter for now
|
||||
../../doc/manual
|
||||
./.
|
||||
@@ -90,7 +91,6 @@ mkMesonDerivation (finalAttrs: {
|
||||
]
|
||||
++ lib.optionals buildHtmlManual [
|
||||
mdbook
|
||||
rsync
|
||||
json-schema-for-humans
|
||||
]
|
||||
++ lib.optionals (!officialRelease && buildHtmlManual) [
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
This section provides some notes on how to start hacking on Nix.
|
||||
To get the latest version of Nix from GitHub:
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo.
|
||||
|
||||
```console
|
||||
$ git clone https://github.com/NixOS/nix.git
|
||||
$ cd nix
|
||||
|
||||
@@ -6,14 +6,7 @@ Additionally, see [Testing Nix](./testing.md) for further instructions on how to
|
||||
|
||||
## Building Nix with Debug Symbols
|
||||
|
||||
In the development shell, set the `mesonBuildType` environment variable to `debug` before configuring the build:
|
||||
|
||||
```console
|
||||
[nix-shell]$ export mesonBuildType=debugoptimized
|
||||
```
|
||||
|
||||
Then, proceed to build Nix as described in [Building Nix](./building.md).
|
||||
This will build Nix with debug symbols, which are essential for effective debugging.
|
||||
In the development shell, `mesonBuildType` is set automatically to `debugoptimized`. This builds Nix with debug symbols, which are essential for effective debugging.
|
||||
|
||||
It is also possible to build without optimization for faster build:
|
||||
|
||||
|
||||
@@ -1,27 +1,21 @@
|
||||
{{#include build-trace-entry-v1-fixed.md}}
|
||||
{{#include build-trace-entry-v2-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### Simple build trace entry
|
||||
|
||||
```json
|
||||
{{#include schema/build-trace-entry-v1/simple.json}}
|
||||
```
|
||||
|
||||
### Build trace entry with dependencies
|
||||
|
||||
```json
|
||||
{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}}
|
||||
{{#include schema/build-trace-entry-v2/simple.json}}
|
||||
```
|
||||
|
||||
### Build trace entry with signature
|
||||
|
||||
```json
|
||||
{{#include schema/build-trace-entry-v1/with-signature.json}}
|
||||
{{#include schema/build-trace-entry-v2/with-signature.json}}
|
||||
```
|
||||
|
||||
<!--
|
||||
## Raw Schema
|
||||
|
||||
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v1.json)
|
||||
-->
|
||||
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json)
|
||||
-->
|
||||
@@ -17,7 +17,7 @@ schemas = [
|
||||
'derivation-v4',
|
||||
'derivation-options-v1',
|
||||
'deriving-path-v1',
|
||||
'build-trace-entry-v1',
|
||||
'build-trace-entry-v2',
|
||||
'build-result-v1',
|
||||
'store-v1',
|
||||
]
|
||||
|
||||
@@ -83,7 +83,7 @@ properties:
|
||||
description: |
|
||||
A mapping from output names to their build trace entries.
|
||||
additionalProperties:
|
||||
"$ref": "build-trace-entry-v1.yaml"
|
||||
"$ref": "build-trace-entry-v2.yaml"
|
||||
|
||||
failure:
|
||||
type: object
|
||||
|
||||
File renamed without changes.
+13
-18
@@ -1,5 +1,5 @@
|
||||
"$schema": "http://json-schema.org/draft-04/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json"
|
||||
title: Build Trace Entry
|
||||
description: |
|
||||
A record of a successful build outcome for a specific derivation output.
|
||||
@@ -11,10 +11,17 @@ description: |
|
||||
> This JSON format is currently
|
||||
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
|
||||
> and subject to change.
|
||||
|
||||
Verision history:
|
||||
|
||||
- Version 1: Original format
|
||||
|
||||
- Version 2: Remove `dependentRealisations`
|
||||
|
||||
type: object
|
||||
required:
|
||||
- id
|
||||
- outPath
|
||||
- dependentRealisations
|
||||
- signatures
|
||||
allOf:
|
||||
- "$ref": "#/$defs/key"
|
||||
@@ -22,9 +29,11 @@ allOf:
|
||||
properties:
|
||||
id: {}
|
||||
outPath: {}
|
||||
dependentRealisations: {}
|
||||
signatures: {}
|
||||
additionalProperties: false
|
||||
additionalProperties:
|
||||
dependentRealisations:
|
||||
description: deprecated field
|
||||
type: object
|
||||
|
||||
"$defs":
|
||||
key:
|
||||
@@ -60,7 +69,6 @@ additionalProperties: false
|
||||
type: object
|
||||
required:
|
||||
- outPath
|
||||
- dependentRealisations
|
||||
- signatures
|
||||
properties:
|
||||
outPath:
|
||||
@@ -69,19 +77,6 @@ additionalProperties: false
|
||||
description: |
|
||||
The path to the store object that resulted from building this derivation for the given output name.
|
||||
|
||||
dependentRealisations:
|
||||
type: object
|
||||
title: Underlying Base Build Trace
|
||||
description: |
|
||||
This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
|
||||
|
||||
Keys are derivation output IDs (same format as the main `id` field).
|
||||
Values are the store paths that those dependencies resolved to.
|
||||
|
||||
As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
|
||||
This is the set of base build trace entries that this derived build trace is derived from.
|
||||
(The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
|
||||
|
||||
patternProperties:
|
||||
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
|
||||
"$ref": "store-path-v1.yaml"
|
||||
@@ -70,7 +70,7 @@ properties:
|
||||
"^[A-Za-z0-9+/]{43}=$":
|
||||
type: object
|
||||
additionalProperties:
|
||||
"$ref": "./build-trace-entry-v1.yaml#/$defs/value"
|
||||
"$ref": "./build-trace-entry-v2.yaml#/$defs/value"
|
||||
additionalProperties: false
|
||||
|
||||
"$defs":
|
||||
|
||||
@@ -279,35 +279,3 @@ This release was made possible by the following 33 contributors:
|
||||
- Henry [**(@cootshk)**](https://github.com/cootshk)
|
||||
- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
|
||||
- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)
|
||||
# Release 2.33.3 (2026-02-13)
|
||||
|
||||
- S3 binary caches now use virtual-hosted-style addressing by default [#15208](https://github.com/NixOS/nix/issues/15208)
|
||||
|
||||
S3 binary caches now use virtual-hosted-style URLs
|
||||
(`https://bucket.s3.region.amazonaws.com/key`) instead of path-style URLs
|
||||
(`https://s3.region.amazonaws.com/bucket/key`) when connecting to standard AWS
|
||||
S3 endpoints. This enables HTTP/2 multiplexing and fixes TCP connection
|
||||
exhaustion (TIME_WAIT socket accumulation) under high-concurrency workloads.
|
||||
|
||||
A new `addressing-style` store option controls this behavior:
|
||||
|
||||
- `auto` (default): virtual-hosted-style for standard AWS endpoints, path-style
|
||||
for custom endpoints.
|
||||
- `path`: forces path-style addressing (deprecated by AWS).
|
||||
- `virtual`: forces virtual-hosted-style addressing (bucket names must not
|
||||
contain dots).
|
||||
|
||||
Bucket names containing dots (e.g., `my.bucket.name`) automatically fall back
|
||||
to path-style addressing in `auto` mode, because dotted names create
|
||||
multi-level subdomains that break TLS wildcard certificate validation.
|
||||
|
||||
Example using path-style for backwards compatibility:
|
||||
|
||||
```
|
||||
s3://my-bucket/key?region=us-east-1&addressing-style=path
|
||||
```
|
||||
|
||||
Additionally, TCP keep-alive is now enabled on all HTTP connections, preventing
|
||||
idle connections from being silently dropped by intermediate network devices
|
||||
(NATs, firewalls, load balancers).
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
let
|
||||
inherit (nixpkgs) lib;
|
||||
|
||||
officialRelease = true;
|
||||
officialRelease = false;
|
||||
|
||||
linux32BitSystems = [ "i686-linux" ];
|
||||
linux64BitSystems = [
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGPtMiwBEAC0sFZW2QW/OaDjKm5zGRpDvHXDsMIUtlHfoi5ce8pocC63W05o
|
||||
FSXbUZjZ1VfYO8lT8DFANCzTkiXYaZx0cPRG2pVY4AOQZDNFt5XrAyvw496XCAIM
|
||||
DTYGFLjCqgjPt9RUFEy4MyHPJTEpB0x3rXgT4ILNu9vsj9Q0vttps7SpbZ3Ldq5H
|
||||
o/BBbLW77q/vNjpYzCbBIXF7ycUGpnNv9Go/WuiDnrBMcyxh+8kjjIHB5cxZSnjJ
|
||||
DUv681+m83v+gLZQGX/jexQrrf5JpS0X9qEnhGLrNUDhtyv5ud3Je4EfamkjLVVC
|
||||
RlNLofgflOCsl/tP80i+K7S1QdKhUALxuJ6H0prYUflGBDxDyC8XYuJ62TT0OUpa
|
||||
vJvgwVlCq8/jq+ykYQXlbuBVOzi5wAuI4l3+HqreSQYPSiwe+6N590Zbafdv1fvN
|
||||
WFtZKCTGMqfyaaAnppioH9/+NWkI2AQxaYVasYM/JEYvY9pJgA7alh51jHW4JglP
|
||||
ErypKfBKPKJID0QENqYoa3bDDCihuNWhgQf9dxzPlj2ckd35Zb6w4DfuSmtjaa9D
|
||||
o0jZVY1JbFuxBqP09+saVPrxLHgmPxjcdzPGQQtAqdO2vyJXNEGLFMoVEZPNaLo3
|
||||
QmcIJnT7oSck+4vGfOYtWUHXQynu/Tnwsv2XkA/uyw8HNe+RRMqv/apnzQARAQAB
|
||||
tCdTZXJnZWkgWmltbWVybWFuIDxzZXJnZWlAemltbWVybWFuLmZvbz6JAlEEEwEK
|
||||
ADsCGwEFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQQVim9TDqIC5fZRYRMU+upj
|
||||
RI4d+QUCaUbsaAIZAQAKCRAU+upjRI4d+VdDD/492HRaJ/8V7R7VUzkafmb2Hb28
|
||||
SLf7oiB8Uq9I7SukiDEaIT1fUhquYWQ9KWpPRNR1TX6ApXnIeuJRMGFoDVIRnmnr
|
||||
cKnYYXfqqc81VxIyKvaumB7KWbS7G4Nbor8AH1ouOOOMMS50OTJOWQA4A26inIuG
|
||||
n+7L8MeS5aT+3uNKDoTKsidC47vnaxNMcke1taPfbfo7vn69PsRCM/g9/7TQYU8b
|
||||
6xp+pM9Ao9nJneRk2YCpsGYRrWTpaik0DFKnfpPKJM/yunhtGLF2IYAp3l1mvHPK
|
||||
nnzo92zjpQuZwazEIK+23V1vRT4IjM2BewbJPAzf2/UuxEjjgNQm0tOtH2JhFNeB
|
||||
VM0BVrGxWrrwrsmv6lWghTtBc6zRWyHrj/rpjtVQNmeKYrHWJeXwVz1rqgGPmB2N
|
||||
k0MZD1UjHHhEs1Cntn7yLmxTPztRJCtR+euRu81Uo2NAvrMJ4xsDjaM0LeLTnzjV
|
||||
9AsPjD188dOFyz7VExZum4+XaaEJ41FIPLEqU3U0GAa6stEy0ylSlIN4x9aiXXVW
|
||||
xfzHHchS5jK6QAjuZxN8t01GRactNylINRf7uoTECFZTtXNqfeuk0HQxBH0LuKVE
|
||||
0PxJbcNI4mVWw1KgTJ8PUVC1IXP3sEpqPdJOYiRXgnpcS26fWOBu0aQ4mhxiaJhr
|
||||
/zBfrkLEqp20TdNDZLkCDQRj7TM1ARAAt73xO24curnHTTgXkkVMMRzcMLx3Mb1a
|
||||
2FuddxC5hzTpEpw01L91UBrXVJEg9K2KAwP5CtCLgPCqXr47Tm7krvHxWwBksgY/
|
||||
6aHRsoPQfCFUZHc0aiO+C4NCzR+aEeGKn66Oc1Hq9oUTpDgiBWhsuEPiyA1OSGF0
|
||||
4L0jeTCqfm68kWp4PIK9yuugkdDsoyj6TonuMsb3V5ctHLqop9KH+eHSkUTPo+Lk
|
||||
+bxaeAOJ1UfbohgbRbrYKAfsaghhOMDH3R1w2pvtUJz+sDbuQsiPFTqbxsXDTFws
|
||||
H4N/AQCYnnvOhqEek2sOEZ19bJXt5UrAr10mX4PGmAkWqE1JWBxpOKG3BXSGOTu1
|
||||
3dFhQfPMK+PmvUrs0kcWQr53K/aRUdKKhIfTcMfkqYTGPK5HclHph24WjXj3QFFA
|
||||
SjksQTdm6486ZmLZK4CTbAFOPfTF/aWg8gu9v4ihdq6lqHNNXxv2xBAChcd59H7p
|
||||
D5zy9z8SpwWR9V5JDmlF6HWIIau1c6lSsQq1xHvYM8EuPe03vJvor+2u/cn5zYF1
|
||||
5ZxAuPI2i5vtavg1s8ZGAAogJ9dVcP36LdJfL9quXWvmovkd//qHIepBB+l/zQio
|
||||
ZRDZlIcfV3Xycaqsb5OqHGARHE0097koipMt5y/iXlqG4Ruue6Idb8bW96EKpaWj
|
||||
kKy/iNfQfQMAEQEAAYkEcgQYAQoAJgIbAhYhBBWKb1MOogLl9lFhExT66mNEjh35
|
||||
BQJpRuz3BQkJHCDCAkDBdCAEGQEKAB0WIQRK3hK0WyJ4BicGpcmpsLVXymMjJQUC
|
||||
Y+0zNQAKCRCpsLVXymMjJbdSD/9+f1FOOeGDAJI6Duo5fsWnf4xJJdtQtDbz6d2A
|
||||
SeDapxeJ3zWfKBD0wu5sISEa0uiWsYSmLtsa2SqVAKHlEaMGRR+tkBMPQ+rvgI4c
|
||||
62YjGTgm+IPd+NFIn+ixFU1hpinTh+KhUEoeOwWCvKs9nZfSG9vkienfiG0bBxo2
|
||||
zrvBzXA50x5hbUL+ghKu/AVfN9qZDwh30O4KZTwk4g4cM9SeaQa4YvHYIS3IEhDZ
|
||||
hGybwrrqV9cs92ln4IJw9WCy9QReBNrdeFgC4+3ziUp1QsG3RvqrtuMttwBVC1Z3
|
||||
bj5QjLLOREhhodfvk98t9yVkragObb4rGrLo1mWuF0c4mJGvXwnrqhCMvzv4M+0T
|
||||
Zdrmw6YpGkGOaOPghVuwoTtqSAkl+zFWIJS89jidvkYG3EqKAkgLKog/TQReCq13
|
||||
HWrF8cMck+Rf2K8k26q/RNZaA9ZUKjLExzz8lsWmd2C7rvkGLrlxnzxz0gGyNR3Q
|
||||
KK74vcPhqeABt2GSkHtEXZFFA9IVVzwlRWK3e0S+mVQnZVjNL+cBPn3/hZHMLesB
|
||||
CucyYZv+DxvT+JkYXBkGSw4s3hpABqGym7gdPUIa0q4rbBFG6xP5sLLBG4yru8vV
|
||||
2dyCMmFqRuxpT49uNfyQ6Vj+dobN6qHnP/9NwfzOixXYBHXR6LBqb/M+iCiJaaIn
|
||||
uiRLHwkQFPrqY0SOHfkQpA//W51vj8meuz7snRO+vZFcjLneFFzqfh1Jdz8IqDpO
|
||||
CkI5pBJmi8e0oSe6r68MkahiQLlYPwm7d+sjHvJhPWipNKWq/uwCgBs+Ac1lpPXR
|
||||
MwLbrZukcLMYlLmb2MrCKmjcMt0BZsZKBNYL3a3X9nHgwXdeqFYS4WQDMCCc09lz
|
||||
9YqfdoEsqRO4qN7D0hFqnwjOzb34ixZ6UO8a8ekY9QKxAgWc9fJWGMg6Pjdg4qsK
|
||||
nqymOIAdGVOJdoRM46wKGVBvbsF2gNfQU4XyzgJo5vHGFwJm6EoSnODlL5e2wsQh
|
||||
uN1oqBt/8ef/plloMEqVBweUBATqSqjRF6IhhYJvWVuQHQL1p1vnV9FebiVj34ir
|
||||
Z8ID+o0AnTJcclbUcDwannGJ0cuDcPhk/v/ahVuoMERCi12qnMBo5B/e6Omyh1yB
|
||||
4pbf4GATGGQipDQG75eC/kP2GQEqJP5WYN0Ar8Le/AA/2xyL7upW0yIByyXCwGEb
|
||||
JRwEgU3+bPyu58bFt8Pftit6J7rA3oBVVMOPrYH5eZwRaj5m2RptwKGL6BfHnhNv
|
||||
ZqmCq9EBGX6L1NI0xHMjEFfXJ8jU01XdfG8nCqkwqsHwslXLhqjJphfHcx89YwbV
|
||||
/15GCuURAv1cKe/7277sOhcvP/QpQqSWgvYExHw8PeFJcTYtF2NrRgNwcQsWS1Rj
|
||||
gXa5Ag0EY+0zcAEQANC5N6kSfezuucAgi+X3BD+MT37mxQyvICSggEJf1LDSmy0+
|
||||
bnvD7setL8CP9etTA2fcVNYKI1oboMyhoCnsRP2jDdv1iXOI/hZg4wSb/D1yUkae
|
||||
fUpxv3Wuci2QKavH2MfraDD7BFMbsQeMcHtn4Rk216T6jndZHnzT1Ih7iX0XeQPb
|
||||
li5fojOiZssgWAVT4HPXFCJB6lI35Hjp35oRYwrtMmu5INinZ79n9h1igGtt1ItZ
|
||||
b7rQKNd772Jxcn4UU71ovORSL/xT5i5sxZ+evQOxkpqUAokMOFaoHcOXLmA1NsFv
|
||||
yryXHK4Ioq9ap2jKlLTWkJWjua9JZ4AmKhbvT8X4ELxIKSCAdJKAWP8ZHbXNu5MD
|
||||
aznyzZQLxSO7uFvu356De75mI5iohZNj5wB5Wju71pBiorTKVj4+iJ4e+xVIzFdG
|
||||
hFC0DehNcl2t9w/y8qHwIQ1yUAjXHLXq0/2jsVeH6bU5q/MsgvUP1jcFe0eyOpxy
|
||||
CDvyFdzZFbI57TnB/fvcZTRZ5ewXMFpH8gzuoFzAjUAP95UjYKgaGdrNPNIy28Ii
|
||||
4zhvdghei2+n9jgiMfcGQg8lyfH5yF0vWWWynX0KcJsRwEZoL2EauVdwq4PcYOoU
|
||||
pQFhpcreCjD4LdZ4yRU4InbhcUogXjrQ9Dz01TbPmQD5b5iso21bCEFBXrhzABEB
|
||||
AAGJAjwEGAEKACYCGwwWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwg
|
||||
hwAKCRAU+upjRI4d+X/XD/sH5xvHPfTJq52v8weFmB52up+DzqG2lyhGdoUQ1Muw
|
||||
dRDLTLXLJrFdfpoOo7/j4Scr0rdc7/dpCn0DLcPuCoPxu+SkjEnVehFmZrGSv7Ga
|
||||
x9dHr3DBh42fdlX/U/EnDuyosY0JU1gNF2/6FIA+bTTOFE3RxfN906RjslYQDjMZ
|
||||
UAlSeLYHOZofdltI0YIr32vrxgdWQGZXPxU4XusDUc0z163OO+TGg7iUNWFZP5Qj
|
||||
ubM7e0YbDX0NPIshk8us99YJmrWnhaix1/W5ryO3DXiGaQ7XFi9u7QofRqvRIctg
|
||||
QXavdepkzJow9V9qpMECAJePIuICq7rm+xy+njjbuF436W7390bfVBwRr+FPADsl
|
||||
jgQP4KvY5rykss30kheom8wNEbveWkhH5oTfH9b7O4KXJfpfJzrlgOWp2BD9JL8t
|
||||
/M4HvFXTr2a75H/QbHK5OFrZeGATuv9OTxv7EZvnrPXU+DYTFldpu7TrNNqKCoj3
|
||||
ZyXmc3Hhg5kskDhfHJppaeOayuhMOpT3ud1MFzROY5SLVIH8rBR12KUgsCUYQcGs
|
||||
Iy0+0QvEGkjb4cAH1NK3VlbqVNsy1RmqRt2B28R2ueewDfTOoqkzt4MmzLqTdnAx
|
||||
mTqmHmkEKhEf3K4MRNUPO2yieUg2COk5l6x9HhAnoxxeOZrTmcMsPY/UViG2HEPm
|
||||
ybkCDQRj7TRDARAA9DZuKdfKq4Bs2+NwxC0aplljWOl8VIsEVg+Q8agD7/HU6/b6
|
||||
Dry0njtWybn2x6Axf/nUdeOC01Fi1lmht/fpj6mRkgAvd/V6P10xnsUoykPSDSTh
|
||||
P25MFFGW3JAA82bwdJ4AJpEQvTZG2nTb3237vlBiI1qHQrac8GYkju2O4UfySRN6
|
||||
7cyi7bMf2pjWBBOEhaNy4b6CMDsb32P/N5J7sTE/TXgrS+u4ITIgjzSrkUkh5Z+B
|
||||
8QVRa7xPIDZJdvZWTEXWu5fgRPZvxbr154GIkWJkFzlDoB1UcO56/uzRUuKhEV6o
|
||||
HW3LMUuWdPMjpHpq8hrL0G2rDniJFUtbDFzHdZK1LUU3T2BJM8rjI3D/euph+IDT
|
||||
27vl5qo72zCYE/iKzx4FMLZcQvx1kUAxkPX8l+dzZEwKeRIIpFDxQvatRtl+z0bM
|
||||
jbkpDb+Yjv66sC4dYRpgTTGX6rok0PWHR3IxDNzyf2j8zQ4LFJ+rVBM1GjGSt6mG
|
||||
j9TeL8CVeiSp4SuJ7I/FJVPHsKb50m+BDzeB31qTydNqh2kKr0DVAUa+TUsCr7e0
|
||||
OYr8WE2adJcRXIW0qw50xXF+W7/05GqSCVD0dpeOUdBTQTsSkQmM3/0hcj9aVo9e
|
||||
UDCM9RF0WRqiDAoHzJFfg+ztamkQI5HO6CklC4Ok22qrHRf6HDNYSuT6QFkAEQEA
|
||||
AYkCPQQYAQoAJwMbIAQWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwf
|
||||
tAAKCRAU+upjRI4d+Y+cD/9yllG6uo934pcHNsVppZBfREFwSc8ywlbosCuSVpay
|
||||
PjSqgrWwDrnqrsk0F2kUdC6rR3BIcXbn+lA9KqylH+cCXAJCkh8EDq6TlQ7Lt5EV
|
||||
w1U0MAMXOyxPwDymQ/BO+iDyjXWkRRYgbF5XiFhCfGeuKyhkhACisAgNZ1uA1P5k
|
||||
0SJYc14YfEhQkB46Y20SpfVHRsQ46FyNB6GHbmTmfoO8La8VTh++7GBdh85HfvkG
|
||||
VNQ3wpi5oXsOLN9+MJOezc0XsW2LQsKQj1/J7QKzGh+lxN5cemsA5aqPzh8dyxeT
|
||||
0lYRFp4AHkimqGUomVpRkbegMIPxXqOE+ZAmsddErw0UtmrKxcmMptOJwNgYzEgu
|
||||
++2vtqerL/NYp+wsdcWaBjCz2F3NiwHgNli7NSB/FPwucZZ5gN5C4SnmeFzrGdHg
|
||||
Oy+tQUN6ayQKljHeBO7CjMlsFNo/dcVrEMa1ShxBMqlj/6ivoEhktLz0Nru4FwNU
|
||||
xE5SJYDYfpjD7Ws8y4LoXgWXjFHrMO6N9GzqLN/e8LT7I+w4ps2MrgJ8QSrelmQ3
|
||||
rjkxp3uWp5v2lqy4rLfpi9iB6zIAeoN2eU1yOM9joxOYMxKYaYeYyP1Mm90wFol8
|
||||
LcTSaN+tVniPddBiL6zvsGBEMbCR9XN3EQ+mErbuw5ovWBOCrr+dvN3FxvD11y4J
|
||||
7w==
|
||||
=mXYP
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -1,51 +0,0 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQENBFZu2zwBCADfatenjH3cvhlU6AeInvp4R0JmPBG942aghFj1Qh57smRcO5Bv
|
||||
y9mqrX3UDdmVvu58V3k1k9/GzPnAG1t+c7ohdymv/AMuNY4pE2sfxx7bX+mncTHX
|
||||
5wthipn8kTNm4WjREjCJM1Bm5sozzEZetED3+0/dWlnHl8b38evnLsD+WbSrDPVp
|
||||
o6M6Eg9IfMwTfcXzdmLmSnGolBWDQ9i1a0x0r3o+sDW5UTnr7jVP+zILcnOZ1Ewl
|
||||
Rn9OJ4Qg3ULM7WTMDYpKH4BO7RLR3aJgmsFAHp17vgUnzzFBZ10MCS3UOyUNoyph
|
||||
xo3belf7Q9nrHcSNbqSeQuBnW/vafAZUreAlABEBAAG0IkVlbGNvIERvbHN0cmEg
|
||||
PGVkb2xzdHJhQGdtYWlsLmNvbT6JATwEEwEIACYCGyMHCwkIBwMCAQYVCAIJCgsE
|
||||
FgIDAQIeAQIXgAUCVm7etAIZAQAKCRCBcLRybXGY3q51B/96qt41tmcDSzrj/UTl
|
||||
O6rErfW5zFvVsJTZ95Duwu87t/DVhw5lKBQcjALqVddufw1nMzyN/tSOMVDW8xe4
|
||||
wMEdcU4+QAMzNX80enuyinsw1glxfLcK0+VbTvqNIfw0sG3MjPqNs6cK2VRfMHK4
|
||||
paJjytBVICszNX9TfjLyIpKKoSSo1vqnT47LDZ5GIMy7l9Cs2sO/rqQHSPcR79yz
|
||||
8m8tbHpDDEMZmJeklckKP2QoiqnHiIvlisDxLclYnUmNaPdaN/f++qZz5Yqvu1n+
|
||||
sNUBA5eLaZH64Uy2SwtABxO3JPJ8nQ2+SFZ7ocFm4Gcdv4aM+Ura9S6fvM91tEJp
|
||||
yAQOiQE5BBMBCAAjBQJWbts8AhsjBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AA
|
||||
CgkQgXC0cm1xmN6sIAgAielxO8zJREqEkA2xudg/o4e9ZlNZ3X1NvY8OzJH/qlB2
|
||||
SmwKqwifhtbC1K0uavXA7eaxdtd2zrI+Yq7IooUyv7juMjHTZhLcFbR5iVkQ4Mfp
|
||||
JmeHXJ/ChYKxD5mMj/C3WbCZ91oCSNZ6Iyi5fvQj/691OC4q+y/2NEUcOI8D8cw8
|
||||
XKHbKtceFYc+nZmdOv3ZZrNTSN/kszGViNNLKgnpPdDVPtLp+vjXtbmitiFG2HL/
|
||||
WfbJ+3Gh2Yr1Vy3O9dWKH++e1AmIv7WWqmUjRFVpqC/wr7/BLaScWT8WKF5vkshU
|
||||
gq8Ez1/cuizsgs3wQIZWgXKQK5njvwnbKg+Zmh/uGbQmRWVsY28gRG9sc3RyYSA8
|
||||
ZWVsY28uZG9sc3RyYUB0d2VhZy5pbz6JAU4EEwEIADgWIQS1QdVTAScOC88Vyl2B
|
||||
cLRybXGY3gUCXELt4gIbIwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCBcLRy
|
||||
bXGY3ujFCADfS5D1xHU8KH6TpqgssSggYVq62Wwn/Ga+4XPPetM+ajcXagyH6SwB
|
||||
mxlHICcnv9xC93ryiTI10P1ADJl+aBsI66wEdHBU+ty4RTDy4JZNUPtmRCk9LhSc
|
||||
mtUO3ry/wtWkRLdJxP49hg7BbQvWoU0M6WODp7SJjPKPWNX64mzHBeOuy+DqGCbM
|
||||
lpGNCvW8ahU/ewbm7+xwWmzqLDoWzXjHsdF4QdzMVM/vkAgWEP4y0wEqFASzIYaR
|
||||
GNEkBWU4OQVq5Bdm9+wWWAgsbM0FJAQl0GDqnz4QxWzxxCAAXdbh9F5ffafWYsA9
|
||||
bise4ZQLkvYo6iUnrcFm4dtZbT8iL3gptCtFZWxjbyBEb2xzdHJhIDxlZWxjby5k
|
||||
b2xzdHJhQGxvZ2ljYmxveC5jb20+iQE5BBMBCAAjBQJWbt6nAhsjBwsJCAcDAgEG
|
||||
FQgCCQoLBBYCAwECHgECF4AACgkQgXC0cm1xmN4b/wf8DApMV/jSPEpibekrUPQu
|
||||
Ye3Z8cxBQuRm/nOPowtPEH/ShAevrCdRiob2nuEZWNoqZ2e5/+6ud07Hs9bslvco
|
||||
cDv1jeY1dof1idxfKhH3kfSpuD2XJhuzQBxBqOrIlCS/rdnW+Y9wOGD7+bs9QpcA
|
||||
IyAeQGLLkfggAxaGYQ2Aev8pS7i3a/+lOWbFhcTe02I49KemCOJqBorG5FfILLNr
|
||||
DjO3EoutNGpuz6rZvc/BlymphWBoAdUmxgoObr7NYWgw9pI8WeE6C7bbSOO7p5aQ
|
||||
spWXU7Hm17DkzsVDpaJlyClllqK+DdKza5oWlBMe/P02jD3Y+0P/2rCCyQQwmH3D
|
||||
RbkBDQRWbts8AQgA0g556xc08dH5YNEjbCwEt1j+XoRnV4+GfbSJIXOl9joIgzRC
|
||||
4IaijvL8+4biWvX7HiybfvBKto0XB1AWLZRC3jWKX5p74I77UAcrD+VQ/roWQqlJ
|
||||
BKbiQMlRYEsj/5Xnf72G90IP4DAFKvNl+rLChe+jUySA91BCtrYoP75Sw1BE9Cyz
|
||||
xEtm4WUzKAJdXI+ZTBttA2Nbqy+GSuzBs7fSKDwREJaZmVrosvmns+pQVG4WPWf4
|
||||
0l4mPguDQmZ9wSWZvBDkpG7AgHYDRYRGkMbAGsVfc6cScN2VsSTa6cbeeAEowKxM
|
||||
qx9RbY3WOq6aKAm0qDvow1nl7WwXwe8K0wQxfQARAQABiQEfBBgBCAAJBQJWbts8
|
||||
AhsMAAoJEIFwtHJtcZjeuAAH/0YNz2Qe1IAEO5oqEZNFOccL4KxVPrBhWUen83/b
|
||||
C6PjOnOqv6q5ztAcms88WIKxBlfzIfq+dzJcbKVS/H7TEXgcaC+7EYW8sJVEsipN
|
||||
BtEZ3LQNJ5coDjm7WZygniah1lfXNuiritAXduK5FWNNndqGArEaeZ8Shzdo/Uyi
|
||||
b9lOsBIL6xc2ZcnX5f+rTu02LCEtEb0FwCycZLEWYf8hG4k8uttIOZOC+CLk/k8d
|
||||
kBmPikMwUVTTV0CdT1cemQKdTaoAaK+kurF6FYXwcnjhRlHrisSt/tVMEwTw4LUM
|
||||
3MYf6qfjjvE4HlDwZal8th7ccoQp/flfJIuRv85xCcKK+PI=
|
||||
=u5cX
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -1,13 +0,0 @@
|
||||
# Maintainer GPG Keys
|
||||
|
||||
Release tags are signed by members of the [Nix maintainer team](https://nixos.org/community/teams/nix/) as part of the [release process](../release-process.md). This directory contains the public GPG keys used for signing.
|
||||
|
||||
## Keys
|
||||
|
||||
- **Eelco Dolstra**
|
||||
GPG Fingerprint: `B541 D553 0127 0E0B CF15 CA5D 8170 B472 6D71 98DE`
|
||||
|
||||
- **Sergei Zimmerman**
|
||||
GPG Fingerprint: [`158A 6F53 0EA2 02E5 F651 6113 14FA EA63 448E 1DF9`](https://keys.openpgp.org/vks/v1/by-fingerprint/158A6F530EA202E5F651611314FAEA63448E1DF9)
|
||||
|
||||
<!-- TODO: Add keys for other Nix team members -->
|
||||
@@ -5,11 +5,11 @@
|
||||
The release process is intended to create the following for each
|
||||
release:
|
||||
|
||||
* A signed Git tag (public keys in `maintainers/keys/`)
|
||||
* A Git tag
|
||||
|
||||
* Binary tarballs in https://releases.nixos.org/?prefix=nix/
|
||||
|
||||
* Docker images (arm64 and amd64 variants, uploaded to DockerHub and GHCR)
|
||||
* Docker images
|
||||
|
||||
* Closures in https://cache.nixos.org
|
||||
|
||||
@@ -104,17 +104,21 @@ release:
|
||||
evaluation ID (e.g. `1780832` in
|
||||
`https://hydra.nixos.org/eval/1780832`).
|
||||
|
||||
* Tag the release:
|
||||
* Tag the release and upload the release artifacts to
|
||||
[`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/):
|
||||
|
||||
```console
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
|
||||
```
|
||||
|
||||
Note: `IS_LATEST=1` causes the `latest-release` branch to be
|
||||
force-updated. This is used by the `nixos.org` website to get the
|
||||
[latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/).
|
||||
|
||||
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
|
||||
TODO: This script requires the right AWS credentials. Document.
|
||||
|
||||
TODO: This script currently requires a
|
||||
`/home/eelco/Dev/nix-pristine`.
|
||||
|
||||
TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
|
||||
|
||||
@@ -177,18 +181,16 @@ release:
|
||||
* Wait for the desired evaluation of the maintenance jobset to finish
|
||||
building.
|
||||
|
||||
* Tag the release
|
||||
* Run
|
||||
|
||||
```console
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
|
||||
```
|
||||
|
||||
Omit `IS_LATEST=1` when creating a point release that is not on the
|
||||
most recent stable branch. This prevents `nixos.org` to going back
|
||||
to an older release.
|
||||
|
||||
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
|
||||
|
||||
* Bump the version number of the release branch as above (e.g. to
|
||||
`2.12.2`).
|
||||
|
||||
|
||||
+82
-128
@@ -1,8 +1,7 @@
|
||||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i perl -p awscli2 perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 perlPackages.GetoptLongDescriptive gnupg1
|
||||
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
|
||||
|
||||
use strict;
|
||||
use Getopt::Long::Descriptive;
|
||||
use Data::Dumper;
|
||||
use File::Basename;
|
||||
use File::Path;
|
||||
@@ -14,30 +13,7 @@ use Net::Amazon::S3;
|
||||
|
||||
delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
|
||||
|
||||
my ($opt, $usage) = describe_options(
|
||||
'%c %o <eval-id>',
|
||||
[ 'skip-docker', 'Skip Docker image upload' ],
|
||||
[ 'skip-git', 'Skip Git tagging' ],
|
||||
[ 'skip-s3', 'Skip S3 upload' ],
|
||||
[ 'docker-owner=s', 'Docker image owner', { default => 'nixos/nix' } ],
|
||||
[ 'project-root=s', 'Pristine git repository path' ],
|
||||
[ 's3-endpoint=s', 'Custom S3 endpoint' ],
|
||||
[ 's3-host=s', 'S3 host', { default => 's3-eu-west-1.amazonaws.com' } ],
|
||||
[],
|
||||
[ 'help|h', 'Show this help message', { shortcircuit => 1 } ],
|
||||
[],
|
||||
[ 'Environment variables:' ],
|
||||
[ 'AWS_ACCESS_KEY_ID' ],
|
||||
[ 'AWS_SECRET_ACCESS_KEY' ],
|
||||
[ 'AWS_SESSION_TOKEN For OIDC' ],
|
||||
[ 'IS_LATEST Set to "1" to mark as latest release' ],
|
||||
);
|
||||
|
||||
print($usage->text), exit if $opt->help;
|
||||
|
||||
my $evalId = $ARGV[0] or do { print STDERR $usage->text; exit 1 };
|
||||
|
||||
die "--project-root is required unless --skip-git is specified\n" unless $opt->skip_git || $opt->project_root;
|
||||
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
|
||||
|
||||
my $releasesBucketName = "nix-releases";
|
||||
my $channelsBucketName = "nix-channels";
|
||||
@@ -86,38 +62,25 @@ File::Path::make_path($narCache);
|
||||
my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
|
||||
|
||||
# S3 setup.
|
||||
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'};
|
||||
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'};
|
||||
my $aws_session_token = $ENV{'AWS_SESSION_TOKEN'};
|
||||
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
|
||||
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
|
||||
|
||||
my ($s3, $releasesBucket, $s3_channels, $channelsBucket);
|
||||
my $s3 = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
retry => 1,
|
||||
host => "s3-eu-west-1.amazonaws.com",
|
||||
});
|
||||
|
||||
unless ($opt->skip_s3) {
|
||||
$aws_access_key_id or die "No AWS_ACCESS_KEY_ID given.";
|
||||
$aws_secret_access_key or die "No AWS_SECRET_ACCESS_KEY given.";
|
||||
my $releasesBucket = $s3->bucket($releasesBucketName) or die;
|
||||
|
||||
$s3 = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
|
||||
retry => 1,
|
||||
host => $opt->s3_host,
|
||||
secure => ($opt->s3_endpoint && $opt->s3_endpoint =~ /^http:/) ? 0 : 1,
|
||||
});
|
||||
my $s3_us = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
retry => 1,
|
||||
});
|
||||
|
||||
$releasesBucket = $s3->bucket($releasesBucketName) or die;
|
||||
|
||||
$s3_channels = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
|
||||
retry => 1,
|
||||
$opt->s3_endpoint ? (host => $opt->s3_host) : (),
|
||||
$opt->s3_endpoint ? (secure => ($opt->s3_endpoint =~ /^http:/) ? 0 : 1) : (),
|
||||
});
|
||||
|
||||
$channelsBucket = $s3_channels->bucket($channelsBucketName) or die;
|
||||
}
|
||||
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
|
||||
|
||||
sub getStorePath {
|
||||
my ($jobName, $output) = @_;
|
||||
@@ -152,12 +115,11 @@ sub copyManual {
|
||||
File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1});
|
||||
}
|
||||
|
||||
my $awsEndpoint = $opt->s3_endpoint ? "--endpoint-url " . $opt->s3_endpoint : "";
|
||||
system("aws $awsEndpoint s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
|
||||
system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
|
||||
or die "syncing manual to S3\n";
|
||||
}
|
||||
|
||||
copyManual unless $opt->skip_s3;
|
||||
copyManual;
|
||||
|
||||
sub downloadFile {
|
||||
my ($jobName, $productNr, $dstName) = @_;
|
||||
@@ -196,12 +158,30 @@ sub downloadFile {
|
||||
return $sha256_expected;
|
||||
}
|
||||
|
||||
# Upload docker images.
|
||||
downloadFile("binaryTarball.i686-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-linux", "1");
|
||||
downloadFile("binaryTarball.aarch64-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-darwin", "1");
|
||||
downloadFile("binaryTarball.aarch64-darwin", "1");
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
downloadFile("installerScript", "1");
|
||||
|
||||
# Upload docker images to dockerhub.
|
||||
my $dockerManifest = "";
|
||||
my $dockerManifestLatest = "";
|
||||
my $haveDocker = 0;
|
||||
|
||||
unless ($opt->skip_docker) {
|
||||
for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
|
||||
my $system = $platforms->[0];
|
||||
my $dockerPlatform = $platforms->[1];
|
||||
@@ -215,8 +195,8 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
|
||||
print STDERR "loading docker image for $dockerPlatform...\n";
|
||||
system("docker load -i $tmpDir/$fn") == 0 or die;
|
||||
|
||||
my $tag = $opt->docker_owner . ":$version-$dockerPlatform";
|
||||
my $latestTag = $opt->docker_owner . ":latest-$dockerPlatform";
|
||||
my $tag = "nixos/nix:$version-$dockerPlatform";
|
||||
my $latestTag = "nixos/nix:latest-$dockerPlatform";
|
||||
|
||||
print STDERR "tagging $version docker image for $dockerPlatform...\n";
|
||||
system("docker tag nix:$version $tag") == 0 or die;
|
||||
@@ -239,94 +219,68 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
|
||||
}
|
||||
|
||||
if ($haveDocker) {
|
||||
my $dockerOwner = $opt->docker_owner;
|
||||
print STDERR "creating multi-platform docker manifest...\n";
|
||||
system("docker manifest rm $dockerOwner:$version");
|
||||
system("docker manifest create $dockerOwner:$version $dockerManifest") == 0 or die;
|
||||
system("docker manifest rm nixos/nix:$version");
|
||||
system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
|
||||
if ($isLatest) {
|
||||
print STDERR "creating latest multi-platform docker manifest...\n";
|
||||
system("docker manifest rm $dockerOwner:latest");
|
||||
system("docker manifest create $dockerOwner:latest $dockerManifestLatest") == 0 or die;
|
||||
system("docker manifest rm nixos/nix:latest");
|
||||
system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
|
||||
}
|
||||
|
||||
print STDERR "pushing multi-platform docker manifest...\n";
|
||||
system("docker manifest push $dockerOwner:$version") == 0 or die;
|
||||
system("docker manifest push nixos/nix:$version") == 0 or die;
|
||||
|
||||
if ($isLatest) {
|
||||
print STDERR "pushing latest multi-platform docker manifest...\n";
|
||||
system("docker manifest push $dockerOwner:latest") == 0 or die;
|
||||
system("docker manifest push nixos/nix:latest") == 0 or die;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Upload nix-fallback-paths.nix.
|
||||
write_file("$tmpDir/fallback-paths.nix",
|
||||
"{\n" .
|
||||
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
|
||||
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
|
||||
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
|
||||
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
|
||||
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
|
||||
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
|
||||
"}\n");
|
||||
|
||||
# Upload release files to S3.
|
||||
unless ($opt->skip_s3) {
|
||||
downloadFile("binaryTarball.i686-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-linux", "1");
|
||||
downloadFile("binaryTarball.aarch64-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-darwin", "1");
|
||||
downloadFile("binaryTarball.aarch64-darwin", "1");
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
downloadFile("installerScript", "1");
|
||||
for my $fn (glob "$tmpDir/*") {
|
||||
my $name = basename($fn);
|
||||
next if $name eq "manual";
|
||||
my $dstKey = "$releaseDir/" . $name;
|
||||
unless (defined $releasesBucket->head_key($dstKey)) {
|
||||
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
|
||||
|
||||
# Upload nix-fallback-paths.nix.
|
||||
write_file("$tmpDir/fallback-paths.nix",
|
||||
"{\n" .
|
||||
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
|
||||
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
|
||||
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
|
||||
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
|
||||
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
|
||||
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
|
||||
"}\n");
|
||||
my $configuration = ();
|
||||
$configuration->{content_type} = "application/octet-stream";
|
||||
|
||||
for my $fn (glob "$tmpDir/*") {
|
||||
my $name = basename($fn);
|
||||
next if $name eq "manual";
|
||||
my $dstKey = "$releaseDir/" . $name;
|
||||
unless (defined $releasesBucket->head_key($dstKey)) {
|
||||
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
|
||||
|
||||
my $configuration = ();
|
||||
$configuration->{content_type} = "application/octet-stream";
|
||||
|
||||
if ($fn =~ /.sha256|install|\.nix$/) {
|
||||
$configuration->{content_type} = "text/plain";
|
||||
}
|
||||
|
||||
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
|
||||
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
|
||||
if ($fn =~ /.sha256|install|\.nix$/) {
|
||||
$configuration->{content_type} = "text/plain";
|
||||
}
|
||||
}
|
||||
|
||||
# Update the "latest" symlink.
|
||||
$channelsBucket->add_key(
|
||||
"nix-latest/install", "",
|
||||
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
|
||||
or die $channelsBucket->err . ": " . $channelsBucket->errstr
|
||||
if $isLatest;
|
||||
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
|
||||
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
|
||||
}
|
||||
}
|
||||
|
||||
# Update the "latest" symlink.
|
||||
$channelsBucket->add_key(
|
||||
"nix-latest/install", "",
|
||||
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
|
||||
or die $channelsBucket->err . ": " . $channelsBucket->errstr
|
||||
if $isLatest;
|
||||
|
||||
# Tag the release in Git.
|
||||
unless ($opt->skip_git) {
|
||||
chdir($opt->project_root) or die "Cannot chdir to " . $opt->project_root . ": $!";
|
||||
system("git remote update origin") == 0 or die;
|
||||
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
|
||||
system("git push origin refs/tags/$version") == 0 or die;
|
||||
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
|
||||
}
|
||||
chdir("/home/eelco/Dev/nix-pristine") or die;
|
||||
system("git remote update origin") == 0 or die;
|
||||
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
|
||||
system("git push --tags") == 0 or die;
|
||||
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
|
||||
|
||||
File::Path::remove_tree($narCache, {safe => 1});
|
||||
File::Path::remove_tree($tmpDir, {safe => 1});
|
||||
@@ -259,10 +259,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
|
||||
# We use this shell with the local checkout, not unpackPhase.
|
||||
src = null;
|
||||
|
||||
# Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html
|
||||
# Remove when gdb fix is rolled out everywhere.
|
||||
separateDebugInfo = false;
|
||||
|
||||
mesonBuildType = "debugoptimized";
|
||||
|
||||
env = {
|
||||
# For `make format`, to work without installing pre-commit
|
||||
_NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml"
|
||||
|
||||
@@ -62,9 +62,11 @@ schemas = [
|
||||
},
|
||||
{
|
||||
'stem' : 'build-trace-entry',
|
||||
'schema' : schema_dir / 'build-trace-entry-v1.yaml',
|
||||
'schema' : schema_dir / 'build-trace-entry-v2.yaml',
|
||||
'files' : [
|
||||
'simple.json',
|
||||
# The field is no longer supported, but we want to show that we
|
||||
# ignore it during parsing.
|
||||
'with-dependent-realisations.json',
|
||||
'with-signature.json',
|
||||
],
|
||||
|
||||
@@ -741,6 +741,11 @@ public:
|
||||
inDebugger = true;
|
||||
}
|
||||
|
||||
DebuggerGuard(DebuggerGuard &&) = delete;
|
||||
DebuggerGuard(const DebuggerGuard &) = delete;
|
||||
DebuggerGuard & operator=(DebuggerGuard &&) = delete;
|
||||
DebuggerGuard & operator=(const DebuggerGuard &) = delete;
|
||||
|
||||
~DebuggerGuard()
|
||||
{
|
||||
inDebugger = false;
|
||||
|
||||
@@ -107,6 +107,8 @@ private:
|
||||
Bindings & operator=(const Bindings &) = delete;
|
||||
Bindings & operator=(Bindings &&) = delete;
|
||||
|
||||
~Bindings() = default;
|
||||
|
||||
friend class BindingsBuilder;
|
||||
|
||||
/**
|
||||
|
||||
@@ -164,8 +164,6 @@ public:
|
||||
Value ** elems;
|
||||
ListBuilder(EvalMemory & mem, size_t size);
|
||||
|
||||
// NOTE: Can be noexcept because we are just copying integral values and
|
||||
// raw pointers.
|
||||
ListBuilder(ListBuilder && x) noexcept
|
||||
: size(x.size)
|
||||
, inlineElems{x.inlineElems[0], x.inlineElems[1]}
|
||||
@@ -173,6 +171,11 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
ListBuilder(const ListBuilder &) = delete;
|
||||
ListBuilder & operator=(ListBuilder &&) = delete;
|
||||
ListBuilder & operator=(const ListBuilder &) = delete;
|
||||
~ListBuilder() = default;
|
||||
|
||||
Value *& operator[](size_t n)
|
||||
{
|
||||
return elems[n];
|
||||
|
||||
+18
-2
@@ -4061,6 +4061,8 @@ static RegisterPrimOp primop_sort({
|
||||
|
||||
1. Transitivity
|
||||
|
||||
If a is less than b and b is less than c, then it follows that a is less than c.
|
||||
|
||||
```nix
|
||||
comparator a b && comparator b c -> comparator a c
|
||||
```
|
||||
@@ -4073,9 +4075,23 @@ static RegisterPrimOp primop_sort({
|
||||
|
||||
1. Transitivity of equivalence
|
||||
|
||||
First, two values a and b are considered equivalent with respect to the comparator if:
|
||||
|
||||
```
|
||||
!comparator a b && !comparator b a
|
||||
```
|
||||
|
||||
In other words, neither is considered "less than" the other.
|
||||
|
||||
Transitivity of equivalence means:
|
||||
|
||||
If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c.
|
||||
|
||||
```nix
|
||||
let equiv = a: b: (!comparator a b && !comparator b a); in
|
||||
equiv a b && equiv b c -> equiv a c
|
||||
let
|
||||
equiv = x: y: (!comparator x y && !comparator y x);
|
||||
in
|
||||
equiv a b && equiv b c -> equiv a c
|
||||
```
|
||||
|
||||
If the *comparator* violates any of these properties, then `builtins.sort`
|
||||
|
||||
@@ -48,7 +48,7 @@ public:
|
||||
|
||||
ref<GitRepo> openRepo()
|
||||
{
|
||||
return GitRepo::openRepo(tmpDir, true, false);
|
||||
return GitRepo::openRepo(tmpDir, {.create = true});
|
||||
}
|
||||
|
||||
std::string getRepoName() const
|
||||
|
||||
@@ -236,9 +236,6 @@ void Input::checkLocks(Input specified, Input & result)
|
||||
if (auto prevNarHash = specified.getNarHash())
|
||||
specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
if (auto narHash = result.getNarHash())
|
||||
result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
|
||||
|
||||
for (auto & field : specified.attrs) {
|
||||
auto field2 = result.attrs.find(field.first);
|
||||
if (field2 != result.attrs.end() && field.second != field2->second)
|
||||
|
||||
@@ -203,16 +203,19 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
|
||||
|
||||
} // extern "C"
|
||||
|
||||
static void initRepoAtomically(std::filesystem::path & path, bool bare)
|
||||
static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options)
|
||||
{
|
||||
if (pathExists(path.string()))
|
||||
return;
|
||||
|
||||
if (!options.create)
|
||||
throw Error("Git repository %s does not exist.", path);
|
||||
|
||||
std::filesystem::path tmpDir = createTempDir(path.parent_path());
|
||||
AutoDelete delTmpDir(tmpDir, true);
|
||||
Repository tmpRepo;
|
||||
|
||||
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), bare))
|
||||
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
|
||||
throw Error("creating Git repository %s: %s", path, git_error_last()->message);
|
||||
try {
|
||||
std::filesystem::rename(tmpDir, path);
|
||||
@@ -234,7 +237,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
/** Location of the repository on disk. */
|
||||
std::filesystem::path path;
|
||||
|
||||
bool bare;
|
||||
Options options;
|
||||
|
||||
/**
|
||||
* libgit2 repository. Note that new objects are not written to disk,
|
||||
@@ -255,18 +258,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
*/
|
||||
git_odb_backend * packBackend = nullptr;
|
||||
|
||||
GitRepoImpl(std::filesystem::path _path, bool create, bool bare, bool packfilesOnly = false)
|
||||
GitRepoImpl(std::filesystem::path _path, Options _options)
|
||||
: path(std::move(_path))
|
||||
, bare(bare)
|
||||
, options(_options)
|
||||
{
|
||||
initLibGit2();
|
||||
|
||||
initRepoAtomically(path, bare);
|
||||
initRepoAtomically(path, options);
|
||||
if (git_repository_open(Setter(repo), path.string().c_str()))
|
||||
throw Error("opening Git repository %s: %s", path, git_error_last()->message);
|
||||
|
||||
ObjectDb odb;
|
||||
if (packfilesOnly) {
|
||||
if (options.packfilesOnly) {
|
||||
/* Create a fresh object database because by default the repo also
|
||||
loose object backends. We are not using any of those for the
|
||||
tarball cache, but libgit2 still does a bunch of unnecessary
|
||||
@@ -295,7 +298,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
if (git_odb_add_backend(odb.get(), mempackBackend, 999))
|
||||
throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
|
||||
|
||||
if (packfilesOnly) {
|
||||
if (options.packfilesOnly) {
|
||||
if (git_repository_set_odb(repo.get(), odb.get()))
|
||||
throw Error("setting Git object database: %s", git_error_last()->message);
|
||||
}
|
||||
@@ -366,7 +369,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
{
|
||||
// TODO: as an optimization, it would be nice to include `this` in the pool.
|
||||
return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
|
||||
return make_ref<GitRepoImpl>(path, false, bare);
|
||||
return make_ref<GitRepoImpl>(path, options);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -712,9 +715,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
}
|
||||
};
|
||||
|
||||
ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare, bool packfilesOnly)
|
||||
ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Options options)
|
||||
{
|
||||
return make_ref<GitRepoImpl>(path, create, bare, packfilesOnly);
|
||||
return make_ref<GitRepoImpl>(path, options);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1432,7 +1435,7 @@ ref<GitRepo> Settings::getTarballCache() const
|
||||
* for optimal packfiles.
|
||||
*/
|
||||
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2";
|
||||
return GitRepo::openRepo(repoDir, /*create=*/true, /*bare=*/true, /*packfilesOnly=*/true);
|
||||
return GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true});
|
||||
}
|
||||
|
||||
} // namespace fetchers
|
||||
@@ -1446,7 +1449,7 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path &
|
||||
if (i != cache->end())
|
||||
return i->second;
|
||||
}
|
||||
auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo();
|
||||
auto workdirInfo = GitRepo::openRepo(path, {})->getWorkdirInfo();
|
||||
_cache.lock()->emplace(path, workdirInfo);
|
||||
return workdirInfo;
|
||||
}
|
||||
|
||||
+8
-13
@@ -637,11 +637,6 @@ struct GitInputScheme : InputScheme
|
||||
url);
|
||||
}
|
||||
|
||||
// If we don't check here for the path existence, then we can give libgit2 any directory
|
||||
// and it will initialize them as git directories.
|
||||
if (!pathExists(path)) {
|
||||
throw Error("The path '%s' does not exist.", path);
|
||||
}
|
||||
repoInfo.location = std::filesystem::absolute(path);
|
||||
} else {
|
||||
if (url.scheme == "file")
|
||||
@@ -703,7 +698,7 @@ struct GitInputScheme : InputScheme
|
||||
if (auto res = cache->lookup(key))
|
||||
return getIntAttr(*res, "lastModified");
|
||||
|
||||
auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev);
|
||||
auto lastModified = GitRepo::openRepo(repoDir, {})->getLastModified(rev);
|
||||
|
||||
cache->upsert(key, {{"lastModified", lastModified}});
|
||||
|
||||
@@ -726,7 +721,7 @@ struct GitInputScheme : InputScheme
|
||||
Activity act(
|
||||
*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg()));
|
||||
|
||||
auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
|
||||
auto revCount = GitRepo::openRepo(repoDir, {})->getRevCount(rev);
|
||||
|
||||
cache->upsert(key, Attrs{{"revCount", revCount}});
|
||||
|
||||
@@ -737,7 +732,7 @@ struct GitInputScheme : InputScheme
|
||||
{
|
||||
auto head = std::visit(
|
||||
overloaded{
|
||||
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path)->getWorkdirRef(); },
|
||||
[&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
|
||||
[&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
|
||||
repoInfo.location);
|
||||
if (!head) {
|
||||
@@ -795,7 +790,7 @@ struct GitInputScheme : InputScheme
|
||||
if (auto repoPath = repoInfo.getPath()) {
|
||||
repoDir = *repoPath;
|
||||
if (!input.getRev())
|
||||
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
|
||||
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir, {})->resolveRef(ref).gitRev());
|
||||
} else {
|
||||
auto repoUrl = std::get<ParsedURL>(repoInfo.location);
|
||||
std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
|
||||
@@ -805,7 +800,7 @@ struct GitInputScheme : InputScheme
|
||||
std::filesystem::create_directories(cacheDir.parent_path());
|
||||
PathLocks cacheDirLock({cacheDir.string()});
|
||||
|
||||
auto repo = GitRepo::openRepo(cacheDir, true, true);
|
||||
auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true});
|
||||
|
||||
// We need to set the origin so resolving submodule URLs works
|
||||
repo->setRemote("origin", repoUrl.to_string());
|
||||
@@ -876,7 +871,7 @@ struct GitInputScheme : InputScheme
|
||||
// the remainder
|
||||
}
|
||||
|
||||
auto repo = GitRepo::openRepo(repoDir);
|
||||
auto repo = GitRepo::openRepo(repoDir, {});
|
||||
|
||||
auto isShallow = repo->isShallow();
|
||||
|
||||
@@ -963,7 +958,7 @@ struct GitInputScheme : InputScheme
|
||||
for (auto & submodule : repoInfo.workdirInfo.submodules)
|
||||
repoInfo.workdirInfo.files.insert(submodule.path);
|
||||
|
||||
auto repo = GitRepo::openRepo(repoPath, false, false);
|
||||
auto repo = GitRepo::openRepo(repoPath, {});
|
||||
|
||||
auto exportIgnore = getExportIgnoreAttr(input);
|
||||
|
||||
@@ -1003,7 +998,7 @@ struct GitInputScheme : InputScheme
|
||||
}
|
||||
|
||||
if (!repoInfo.workdirInfo.isDirty) {
|
||||
auto repo = GitRepo::openRepo(repoPath);
|
||||
auto repo = GitRepo::openRepo(repoPath, {});
|
||||
|
||||
if (auto ref = repo->getWorkdirRef())
|
||||
input.attrs.insert_or_assign("ref", *ref);
|
||||
|
||||
@@ -32,8 +32,14 @@ struct GitRepo
|
||||
{
|
||||
virtual ~GitRepo() {}
|
||||
|
||||
static ref<GitRepo>
|
||||
openRepo(const std::filesystem::path & path, bool create = false, bool bare = false, bool packfilesOnly = false);
|
||||
struct Options
|
||||
{
|
||||
bool create = false;
|
||||
bool bare = false;
|
||||
bool packfilesOnly = false;
|
||||
};
|
||||
|
||||
static ref<GitRepo> openRepo(const std::filesystem::path & path, Options options);
|
||||
|
||||
virtual uint64_t getRevCount(const Hash & rev) = 0;
|
||||
|
||||
|
||||
@@ -128,7 +128,6 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
|
||||
state.forceAttrs(*args[0], noPos, "while evaluating the argument passed to builtins.flakeRefToString");
|
||||
fetchers::Attrs attrs;
|
||||
for (const auto & attr : *args[0]->attrs()) {
|
||||
state.forceValue(*attr.value, attr.pos);
|
||||
auto t = attr.value->type();
|
||||
if (t == nInt) {
|
||||
auto intValue = attr.value->integer().value;
|
||||
|
||||
@@ -88,25 +88,38 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VERSION, VALUE); \
|
||||
} \
|
||||
TEST_F(FIXTURE, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VERSION, VALUE); \
|
||||
#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VERSION, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(FIXTURE, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VERSION, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
|
||||
|
||||
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
|
||||
|
||||
} // namespace nix
|
||||
@@ -47,24 +47,30 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(CommonProtoTest, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(CommonProtoTest, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(CommonProtoTest, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
string,
|
||||
"string",
|
||||
@@ -141,7 +147,7 @@ CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
READ_CHARACTERIZATION_TEST(
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
(std::tuple<Realisation>{
|
||||
@@ -149,16 +155,6 @@ CHARACTERIZATION_TEST(
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
.dependentRealisations =
|
||||
{
|
||||
{
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the output of a CA derivation",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"narSize": 152,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "test-ca-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {},
|
||||
"derivations": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "test-ca-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the output of a CA derivation",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"narSize": 152,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"narSize": 232,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": [
|
||||
"out"
|
||||
]
|
||||
}
|
||||
},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "root-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
},
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "dep-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {},
|
||||
"derivations": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": [
|
||||
"out"
|
||||
]
|
||||
}
|
||||
},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "root-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
},
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "dep-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"narSize": 232,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
},
|
||||
"w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the dependency output",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
|
||||
"narSize": 144,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": {
|
||||
"contents": {
|
||||
"contents": "Hello, world!",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
|
||||
"narSize": 128,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
|
||||
"contents": {
|
||||
"contents": "I am a dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"narSize": 136,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
},
|
||||
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
|
||||
"contents": {
|
||||
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"narSize": 184,
|
||||
"references": [
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
|
||||
],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
|
||||
"contents": {
|
||||
"contents": "I am a dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"narSize": 136,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
},
|
||||
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
|
||||
"contents": {
|
||||
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"narSize": 184,
|
||||
"references": [
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
|
||||
],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
#include "nix/store/derivations.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/util/experimental-features.hh"
|
||||
#include "nix/util/environment-variables.hh"
|
||||
#include "nix/util/tests/test-data.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
@@ -50,11 +50,7 @@ static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::st
|
||||
}
|
||||
|
||||
// Register benchmarks for actual test derivation files if they exist
|
||||
BENCHMARK_CAPTURE(
|
||||
BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
|
||||
BENCHMARK_CAPTURE(
|
||||
BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
|
||||
BENCHMARK_CAPTURE(
|
||||
BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
|
||||
BENCHMARK_CAPTURE(
|
||||
BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
|
||||
BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
|
||||
BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());
|
||||
BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
|
||||
BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());
|
||||
@@ -85,6 +85,7 @@ sources = files(
|
||||
'store-reference.cc',
|
||||
'uds-remote-store.cc',
|
||||
'worker-protocol.cc',
|
||||
'worker-substitution.cc',
|
||||
'write-derivation.cc',
|
||||
)
|
||||
|
||||
@@ -123,6 +124,7 @@ if get_option('benchmarks')
|
||||
'bench-main.cc',
|
||||
'derivation-parser-bench.cc',
|
||||
'ref-scan-bench.cc',
|
||||
'register-valid-paths-bench.cc',
|
||||
)
|
||||
|
||||
benchmark_exe = executable(
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "nix/store/tests/nix_api_store.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
#include "nix/util/tests/string_callback.hh"
|
||||
#include "nix/util/tests/test-data.hh"
|
||||
#include "nix/util/url.hh"
|
||||
|
||||
#include "store-tests-config.hh"
|
||||
@@ -302,7 +303,7 @@ public:
|
||||
|
||||
store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
@@ -357,7 +358,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
@@ -404,7 +405,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
@@ -449,7 +450,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
@@ -494,7 +495,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
|
||||
|
||||
auto * store = open_local_store();
|
||||
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
@@ -870,7 +871,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagati
|
||||
*/
|
||||
static std::string load_json_from_test_data(const char * filename)
|
||||
{
|
||||
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
|
||||
std::filesystem::path unitTestData = nix::getUnitTestData();
|
||||
std::ifstream t{unitTestData / filename};
|
||||
std::stringstream buffer;
|
||||
buffer << t.rdbuf();
|
||||
|
||||
@@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json)
|
||||
writeJsonTest(name, value);
|
||||
}
|
||||
|
||||
Realisation simple{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
RealisationJSON,
|
||||
RealisationJsonTest,
|
||||
([] {
|
||||
Realisation simple{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
};
|
||||
return ::testing::Values(
|
||||
std::pair{
|
||||
"simple",
|
||||
simple,
|
||||
},
|
||||
std::pair{
|
||||
"with-signature",
|
||||
[&] {
|
||||
auto r = simple;
|
||||
// FIXME actually sign properly
|
||||
r.signatures = {"asdfasdfasdf"};
|
||||
return r;
|
||||
}()},
|
||||
std::pair{
|
||||
"with-dependent-realisations",
|
||||
[&] {
|
||||
auto r = simple;
|
||||
r.dependentRealisations = {{
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
}};
|
||||
return r;
|
||||
}(),
|
||||
});
|
||||
}
|
||||
::testing::Values(
|
||||
std::pair{
|
||||
"simple",
|
||||
simple,
|
||||
},
|
||||
std::pair{
|
||||
"with-signature",
|
||||
[&] {
|
||||
auto r = simple;
|
||||
// FIXME actually sign properly
|
||||
r.signatures = {"asdfasdfasdf"};
|
||||
return r;
|
||||
}(),
|
||||
}));
|
||||
|
||||
()));
|
||||
/**
|
||||
* We no longer have a notion of "dependent realisations", but we still
|
||||
* want to parse old realisation files. So make this just be a read test
|
||||
* (no write direction), accordingly.
|
||||
*/
|
||||
TEST_F(RealisationTest, dependent_realisations_from_json)
|
||||
{
|
||||
readJsonTest("with-dependent-realisations", simple);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -0,0 +1,79 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "nix/store/derivations.hh"
|
||||
#include "nix/store/local-store.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/hash.hh"
|
||||
#include "nix/util/tests/test-data.hh"
|
||||
|
||||
#ifndef _WIN32
|
||||
|
||||
# include <filesystem>
|
||||
# include <fstream>
|
||||
|
||||
using namespace nix;
|
||||
|
||||
static void BM_RegisterValidPathsDerivations(benchmark::State & state)
|
||||
{
|
||||
const int derivationCount = state.range(0);
|
||||
|
||||
for (auto _ : state) {
|
||||
state.PauseTiming();
|
||||
|
||||
auto tmpRoot = createTempDir();
|
||||
auto realStoreDir = tmpRoot / "nix/store";
|
||||
std::filesystem::create_directories(realStoreDir);
|
||||
|
||||
std::shared_ptr<Store> store = openStore(fmt("local?root=%s", tmpRoot.string()));
|
||||
auto localStore = std::dynamic_pointer_cast<LocalStore>(store);
|
||||
if (!localStore)
|
||||
throw Error("expected local store");
|
||||
|
||||
ValidPathInfos infos;
|
||||
for (int i = 0; i < derivationCount; ++i) {
|
||||
std::string drvName = fmt("register-valid-paths-bench-%d", i);
|
||||
auto drvPath = StorePath::random(drvName + ".drv");
|
||||
|
||||
Derivation drv;
|
||||
drv.name = drvName;
|
||||
drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}});
|
||||
drv.platform = "x86_64-linux";
|
||||
drv.builder = "foo";
|
||||
drv.env["out"] = "";
|
||||
drv.fillInOutputPaths(*localStore);
|
||||
|
||||
auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false);
|
||||
|
||||
/* Create an on-disk store object without registering it
|
||||
in the SQLite DB. LocalFSStore::getFSAccessor(path, false)
|
||||
allows reading store objects based on their filesystem
|
||||
presence alone. */
|
||||
std::ofstream out(realStoreDir / std::string(drvPath.to_string()), std::ios::binary);
|
||||
out.write(drvContents.data(), drvContents.size());
|
||||
if (!out)
|
||||
throw SysError("writing derivation to store");
|
||||
|
||||
ValidPathInfo info{drvPath, UnkeyedValidPathInfo(*localStore, Hash::dummy)};
|
||||
info.narSize = drvContents.size();
|
||||
|
||||
infos.emplace(drvPath, std::move(info));
|
||||
}
|
||||
|
||||
state.ResumeTiming();
|
||||
|
||||
localStore->registerValidPaths(infos);
|
||||
|
||||
state.PauseTiming();
|
||||
localStore.reset();
|
||||
store.reset();
|
||||
std::filesystem::remove_all(tmpRoot);
|
||||
state.ResumeTiming();
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * derivationCount);
|
||||
}
|
||||
|
||||
BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10);
|
||||
|
||||
#endif
|
||||
+15
-237
@@ -104,33 +104,6 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
},
|
||||
"with_absolute_endpoint_uri",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=virtual",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"with_addressing_style_virtual",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=path",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
"with_addressing_style_path",
|
||||
},
|
||||
ParsedS3URLTestCase{
|
||||
"s3://bucket/key?addressing-style=auto",
|
||||
{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Auto,
|
||||
},
|
||||
"with_addressing_style_auto",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<ParsedS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
@@ -165,26 +138,6 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}),
|
||||
[](const ::testing::TestParamInfo<InvalidS3URLTestCase> & info) { return info.param.description; });
|
||||
|
||||
TEST(ParsedS3URLTest, invalidAddressingStyleThrows)
|
||||
{
|
||||
ASSERT_THROW(ParsedS3URL::parse(parseURL("s3://bucket/key?addressing-style=bogus")), InvalidS3AddressingStyle);
|
||||
}
|
||||
|
||||
TEST(ParsedS3URLTest, virtualStyleWithAuthoritylessEndpointThrows)
|
||||
{
|
||||
ParsedS3URL input{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "file",
|
||||
.path = {"", "some", "path"},
|
||||
},
|
||||
};
|
||||
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// S3 URL to HTTPS Conversion Tests
|
||||
// =============================================================================
|
||||
@@ -213,7 +166,6 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversion,
|
||||
S3ToHttpsConversionTest,
|
||||
::testing::Values(
|
||||
// Default (auto) addressing style: virtual-hosted for standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
@@ -221,10 +173,10 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-key.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
},
|
||||
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt",
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt",
|
||||
"basic_s3_default_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -235,13 +187,12 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "prod-cache.s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "nix", "store", "abc123.nar.xz"},
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"},
|
||||
},
|
||||
"https://prod-cache.s3.eu-west-1.amazonaws.com/nix/store/abc123.nar.xz",
|
||||
"https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz",
|
||||
"with_eu_west_1_region",
|
||||
},
|
||||
// Custom endpoint authority: path-style by default
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
@@ -257,7 +208,6 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
"http://custom.s3.com/bucket/key",
|
||||
"custom_endpoint_authority",
|
||||
},
|
||||
// Custom endpoint URL: path-style by default
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
@@ -286,10 +236,10 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = {"", "path", "to", "file.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"},
|
||||
.path = {"", "bucket", "path", "to", "file.txt"},
|
||||
},
|
||||
"https://bucket.s3.ap-southeast-2.amazonaws.com/path/to/file.txt",
|
||||
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
|
||||
"complex_path_and_region",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -300,11 +250,11 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "my-bucket.s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-key.txt"},
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
.query = {{"versionId", "abc123xyz"}},
|
||||
},
|
||||
"https://my-bucket.s3.us-east-1.amazonaws.com/my-key.txt?versionId=abc123xyz",
|
||||
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
|
||||
"with_versionId",
|
||||
},
|
||||
S3ToHttpsConversionTestCase{
|
||||
@@ -316,185 +266,13 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "versioned-bucket.s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "path", "to", "object"},
|
||||
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
|
||||
.path = {"", "versioned-bucket", "path", "to", "object"},
|
||||
.query = {{"versionId", "version456"}},
|
||||
},
|
||||
"https://versioned-bucket.s3.eu-west-1.amazonaws.com/path/to/object?versionId=version456",
|
||||
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
|
||||
"with_region_and_versionId",
|
||||
},
|
||||
// Explicit addressing-style=path forces path-style on standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my-bucket",
|
||||
.key = {"my-key.txt"},
|
||||
.region = "us-west-2",
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-west-2.amazonaws.com"},
|
||||
.path = {"", "my-bucket", "my-key.txt"},
|
||||
},
|
||||
"https://s3.us-west-2.amazonaws.com/my-bucket/my-key.txt",
|
||||
"explicit_path_style",
|
||||
},
|
||||
// Explicit addressing-style=virtual forces virtual-hosted-style on custom endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.scheme = "http",
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = "custom.s3.com"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.custom.s3.com"},
|
||||
.path = {"", "key"},
|
||||
},
|
||||
"http://bucket.custom.s3.com/key",
|
||||
"explicit_virtual_style_custom_endpoint",
|
||||
},
|
||||
// Explicit addressing-style=virtual with full endpoint URL
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "server", .port = 9000},
|
||||
.path = {""},
|
||||
},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "bucket.server", .port = 9000},
|
||||
.path = {"", "key"},
|
||||
},
|
||||
"http://bucket.server:9000/key",
|
||||
"explicit_virtual_style_full_endpoint_url",
|
||||
},
|
||||
// Dotted bucket names work normally with explicit path-style
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Path,
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my.bucket", "key"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my.bucket/key",
|
||||
"dotted_bucket_with_path_style",
|
||||
},
|
||||
// Dotted bucket names fall back to path-style with auto on standard AWS endpoints
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket.name",
|
||||
.key = {"key"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
|
||||
.path = {"", "my.bucket.name", "key"},
|
||||
},
|
||||
"https://s3.us-east-1.amazonaws.com/my.bucket.name/key",
|
||||
"dotted_bucket_with_auto_style_on_aws",
|
||||
},
|
||||
// Dotted bucket names work with auto style on custom endpoints (auto = path-style)
|
||||
S3ToHttpsConversionTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.endpoint = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
ParsedURL{
|
||||
.scheme = "https",
|
||||
.authority = ParsedURL::Authority{.host = "minio.local"},
|
||||
.path = {"", "my.bucket", "key"},
|
||||
},
|
||||
"https://minio.local/my.bucket/key",
|
||||
"dotted_bucket_with_auto_style_custom_endpoint",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
|
||||
|
||||
// =============================================================================
|
||||
// S3 URL to HTTPS Conversion Error Tests
|
||||
// =============================================================================
|
||||
|
||||
struct S3ToHttpsConversionErrorTestCase
|
||||
{
|
||||
ParsedS3URL input;
|
||||
std::string description;
|
||||
};
|
||||
|
||||
class S3ToHttpsConversionErrorTest : public ::testing::WithParamInterface<S3ToHttpsConversionErrorTestCase>,
|
||||
public ::testing::Test
|
||||
{};
|
||||
|
||||
TEST_P(S3ToHttpsConversionErrorTest, ThrowsOnConversion)
|
||||
{
|
||||
auto & [input, description] = GetParam();
|
||||
ASSERT_THROW(input.toHttpsUrl(), nix::Error);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
S3ToHttpsConversionErrors,
|
||||
S3ToHttpsConversionErrorTest,
|
||||
::testing::Values(
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = ""},
|
||||
},
|
||||
"virtual_style_with_empty_host_authority",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket.name",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_multi_dot",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint = ParsedURL::Authority{.host = "minio.local"},
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_custom_authority",
|
||||
},
|
||||
S3ToHttpsConversionErrorTestCase{
|
||||
ParsedS3URL{
|
||||
.bucket = "my.bucket",
|
||||
.key = {"key"},
|
||||
.addressingStyle = S3AddressingStyle::Virtual,
|
||||
.endpoint =
|
||||
ParsedURL{
|
||||
.scheme = "http",
|
||||
.authority = ParsedURL::Authority{.host = "minio.local", .port = 9000},
|
||||
.path = {""},
|
||||
},
|
||||
},
|
||||
"dotted_bucket_with_explicit_virtual_style_full_endpoint_url",
|
||||
}),
|
||||
[](const ::testing::TestParamInfo<S3ToHttpsConversionErrorTestCase> & info) { return info.param.description; });
|
||||
|
||||
} // namespace nix
|
||||
@@ -118,7 +118,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
@@ -128,16 +128,6 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
.dependentRealisations =
|
||||
{
|
||||
{
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
@@ -171,7 +171,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
@@ -181,16 +181,6 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
.dependentRealisations =
|
||||
{
|
||||
{
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/derivations.hh"
|
||||
#include "nix/store/dummy-store-impl.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
|
||||
#include "nix/store/tests/libstore.hh"
|
||||
#include "nix/util/tests/json-characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest<ref<DummyStore>>
|
||||
{
|
||||
std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution";
|
||||
|
||||
protected:
|
||||
ref<DummyStore> dummyStore;
|
||||
ref<DummyStore> substituter;
|
||||
|
||||
WorkerSubstitutionTest()
|
||||
: LibStoreTest([] {
|
||||
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
|
||||
config->readOnly = false;
|
||||
return config->openDummyStore();
|
||||
}())
|
||||
, dummyStore(store.dynamic_pointer_cast<DummyStore>())
|
||||
, substituter([] {
|
||||
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
|
||||
config->readOnly = false;
|
||||
config->isTrusted = true;
|
||||
return config->openDummyStore();
|
||||
}())
|
||||
{
|
||||
}
|
||||
|
||||
public:
|
||||
std::filesystem::path goldenMaster(std::string_view testStem) const override
|
||||
{
|
||||
return unitTestData / testStem;
|
||||
}
|
||||
|
||||
static void SetUpTestSuite()
|
||||
{
|
||||
initLibStore(false);
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(WorkerSubstitutionTest, singleStoreObject)
|
||||
{
|
||||
// Add a store path to the substituter
|
||||
auto pathInSubstituter = substituter->addToStore(
|
||||
"hello",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "Hello, world!",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Snapshot the substituter (has one store object)
|
||||
checkpointJson("single/substituter", substituter);
|
||||
|
||||
// Snapshot the destination store before (should be empty)
|
||||
checkpointJson("../dummy-store/empty", dummyStore);
|
||||
|
||||
// The path should not exist in the destination store yet
|
||||
ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituerAsStore = substituter;
|
||||
worker.getSubstituters = [substituerAsStore]() -> std::list<ref<Store>> { return {substituerAsStore}; };
|
||||
|
||||
// Create a substitution goal for the path
|
||||
auto goal = worker.makePathSubstitutionGoal(pathInSubstituter);
|
||||
|
||||
// Run the worker with -j0 semantics (no local builds, only substitution)
|
||||
// The worker.run() takes a set of goals
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after (should match the substituter)
|
||||
checkpointJson("single/substituter", dummyStore);
|
||||
|
||||
// The path should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter));
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
}
|
||||
|
||||
TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject)
|
||||
{
|
||||
// First, add a dependency store path to the substituter
|
||||
auto dependencyPath = substituter->addToStore(
|
||||
"dependency",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I am a dependency",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Now add a store path that references the dependency
|
||||
auto mainPath = substituter->addToStore(
|
||||
"main",
|
||||
SourcePath{
|
||||
[&] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
// Include a reference to the dependency path in the contents
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I depend on " + substituter->printStorePath(dependencyPath),
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256,
|
||||
StorePathSet{dependencyPath});
|
||||
|
||||
// Snapshot the substituter (has two store objects)
|
||||
checkpointJson("with-dep/substituter", substituter);
|
||||
|
||||
// Snapshot the destination store before (should be empty)
|
||||
checkpointJson("../dummy-store/empty", dummyStore);
|
||||
|
||||
// Neither path should exist in the destination store yet
|
||||
ASSERT_FALSE(dummyStore->isValidPath(dependencyPath));
|
||||
ASSERT_FALSE(dummyStore->isValidPath(mainPath));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituterAsStore = substituter;
|
||||
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||
|
||||
// Create a substitution goal for the main path only
|
||||
// The worker should automatically substitute the dependency as well
|
||||
auto goal = worker.makePathSubstitutionGoal(mainPath);
|
||||
|
||||
// Run the worker
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after (should match the substituter)
|
||||
checkpointJson("with-dep/substituter", dummyStore);
|
||||
|
||||
// Both paths should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(dependencyPath));
|
||||
ASSERT_TRUE(dummyStore->isValidPath(mainPath));
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
}
|
||||
|
||||
TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
{
|
||||
// Enable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
|
||||
// Create a CA floating output derivation
|
||||
Derivation drv;
|
||||
drv.name = "test-ca-drv";
|
||||
drv.outputs = {
|
||||
{
|
||||
"out",
|
||||
DerivationOutput{DerivationOutput::CAFloating{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hashAlgo = HashAlgorithm::SHA256,
|
||||
}},
|
||||
},
|
||||
};
|
||||
|
||||
// Write the derivation to the destination store
|
||||
auto drvPath = writeDerivation(*dummyStore, drv);
|
||||
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("ca-drv/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo of the derivation
|
||||
// For CA floating derivations, the kind is Deferred since outputs aren't known until build
|
||||
auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
|
||||
ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto drvHash = hashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object
|
||||
auto outputPath = substituter->addToStore(
|
||||
"test-ca-drv-out",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I am the output of a CA derivation",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Add the realisation (build trace) to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
drvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
UnkeyedRealisation{
|
||||
.outPath = outputPath,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Snapshot the substituter
|
||||
checkpointJson("ca-drv/substituter", substituter);
|
||||
|
||||
// The realisation should not exist in the destination store yet
|
||||
DrvOutput drvOutput{drvHash, "out"};
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituterAsStore = substituter;
|
||||
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||
|
||||
// Create a derivation goal for the CA derivation output
|
||||
// The worker should substitute the output rather than building
|
||||
auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true);
|
||||
|
||||
// Run the worker
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after
|
||||
checkpointJson("ca-drv/store-after", dummyStore);
|
||||
|
||||
// The output path should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(outputPath));
|
||||
|
||||
// The realisation should now exist in the destination store
|
||||
auto realisation = dummyStore->queryRealisation(drvOutput);
|
||||
ASSERT_TRUE(realisation);
|
||||
ASSERT_EQ(realisation->outPath, outputPath);
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
|
||||
// Disable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Test for issue #11928: substituting a CA derivation output should not
|
||||
* require fetching the output of an input derivation when that output
|
||||
* is not referenced.
|
||||
*/
|
||||
TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
{
|
||||
// Enable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
|
||||
// Create the dependency CA floating derivation
|
||||
Derivation depDrv;
|
||||
depDrv.name = "dep-drv";
|
||||
depDrv.outputs = {
|
||||
{
|
||||
"out",
|
||||
DerivationOutput{DerivationOutput::CAFloating{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hashAlgo = HashAlgorithm::SHA256,
|
||||
}},
|
||||
},
|
||||
};
|
||||
|
||||
// Write the dependency derivation to the destination store
|
||||
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
|
||||
|
||||
// Compute the hash modulo for the dependency derivation
|
||||
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
|
||||
ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto depDrvHash = depHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the dependency in the substituter
|
||||
auto depOutputPath = substituter->addToStore(
|
||||
"dep-drv-out",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I am the dependency output",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Add the realisation for the dependency to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
depDrvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
UnkeyedRealisation{
|
||||
.outPath = depOutputPath,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Create the root CA floating derivation that depends on depDrv
|
||||
Derivation rootDrv;
|
||||
rootDrv.name = "root-drv";
|
||||
rootDrv.outputs = {
|
||||
{
|
||||
"out",
|
||||
DerivationOutput{DerivationOutput::CAFloating{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hashAlgo = HashAlgorithm::SHA256,
|
||||
}},
|
||||
},
|
||||
};
|
||||
// Add the dependency derivation as an input
|
||||
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
|
||||
|
||||
// Write the root derivation to the destination store
|
||||
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
|
||||
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("issue-11928/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo for the root derivation
|
||||
auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
|
||||
ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto rootDrvHash = rootHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the root derivation
|
||||
// Note: it does NOT reference the dependency's output
|
||||
auto rootOutputPath = substituter->addToStore(
|
||||
"root-drv-out",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents =
|
||||
"I am the root output. "
|
||||
"I don't reference anything because the other derivation's output is just needed at build time.",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// The DrvOutputs for both derivations
|
||||
DrvOutput depDrvOutput{depDrvHash, "out"};
|
||||
DrvOutput rootDrvOutput{rootDrvHash, "out"};
|
||||
|
||||
// Add the realisation for the root derivation to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
rootDrvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
UnkeyedRealisation{
|
||||
.outPath = rootOutputPath,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Snapshot the substituter
|
||||
// Note: it has realisations for both drvs, but only the root's output store object
|
||||
checkpointJson("issue-11928/substituter", substituter);
|
||||
|
||||
// The realisations should not exist in the destination store yet
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput));
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituterAsStore = substituter;
|
||||
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||
|
||||
// Create a derivation goal for the root derivation output
|
||||
// The worker should substitute the output rather than building
|
||||
auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false);
|
||||
|
||||
// Run the worker
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after
|
||||
checkpointJson("issue-11928/store-after", dummyStore);
|
||||
|
||||
// The root output path should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath));
|
||||
|
||||
// The root realisation should now exist in the destination store
|
||||
auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput);
|
||||
ASSERT_TRUE(rootRealisation);
|
||||
ASSERT_EQ(rootRealisation->outPath, rootOutputPath);
|
||||
|
||||
// #11928: The dependency's REALISATION should be fetched, because
|
||||
// it is needed to resolve the underlying derivation. Currently the
|
||||
// realisation is not fetched (bug). Once fixed: Change
|
||||
// depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the
|
||||
// ASSERT_EQ
|
||||
auto depRealisation = dummyStore->queryRealisation(depDrvOutput);
|
||||
ASSERT_FALSE(depRealisation);
|
||||
// ASSERT_EQ(depRealisation->outPath, depOutputPath);
|
||||
|
||||
// The dependency's OUTPUT is correctly not fetched (not referenced by root output)
|
||||
ASSERT_FALSE(dummyStore->isValidPath(depOutputPath));
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
|
||||
// Disable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "");
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
+12
-103
@@ -13,11 +13,6 @@
|
||||
// C library headers for SSO provider support
|
||||
# include <aws/auth/credentials.h>
|
||||
|
||||
// C library headers for custom logging
|
||||
# include <aws/common/logging.h>
|
||||
|
||||
# include <cstdarg>
|
||||
|
||||
# include <boost/unordered/concurrent_flat_map.hpp>
|
||||
|
||||
# include <chrono>
|
||||
@@ -35,101 +30,6 @@ AwsAuthError::AwsAuthError(int errorCode)
|
||||
|
||||
namespace {
|
||||
|
||||
/**
|
||||
* Map AWS log level to Nix verbosity.
|
||||
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
|
||||
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
|
||||
*
|
||||
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
|
||||
* considers "info" includes low-level details like "Initializing epoll" and
|
||||
* "Starting event-loop thread". What it considers "errors" includes expected
|
||||
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
|
||||
*
|
||||
* To avoid spamming users, we only show FATAL at default verbosity. Everything
|
||||
* else requires -vvvvv (lvlDebug) or higher to see.
|
||||
*/
|
||||
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
|
||||
{
|
||||
switch (level) {
|
||||
case AWS_LL_FATAL:
|
||||
return lvlError;
|
||||
case AWS_LL_ERROR:
|
||||
case AWS_LL_WARN:
|
||||
case AWS_LL_INFO:
|
||||
return lvlDebug;
|
||||
case AWS_LL_DEBUG:
|
||||
case AWS_LL_TRACE:
|
||||
return lvlVomit;
|
||||
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
|
||||
case AWS_LL_NONE:
|
||||
case AWS_LL_COUNT:
|
||||
return lvlDebug;
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom AWS logger that routes logs through Nix's logging infrastructure.
|
||||
*
|
||||
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
|
||||
* or filename-based logging. The underlying C library supports custom loggers
|
||||
* via aws_logger struct with a vtable containing callback functions.
|
||||
*/
|
||||
static int nixAwsLoggerLog(
|
||||
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
|
||||
{
|
||||
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
|
||||
if (nixLevel > verbosity)
|
||||
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
std::array<char, 4096> buffer{};
|
||||
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
|
||||
va_end(args);
|
||||
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
|
||||
return AWS_OP_SUCCESS;
|
||||
|
||||
const char * subjectName = aws_log_subject_name(subject);
|
||||
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
|
||||
return AWS_OP_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current log level for a subject - determines which messages will be logged.
|
||||
* Must be consistent with awsLogLevelToVerbosity mapping.
|
||||
*/
|
||||
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
|
||||
{
|
||||
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
|
||||
if (verbosity >= lvlVomit)
|
||||
return AWS_LL_TRACE;
|
||||
if (verbosity >= lvlDebug)
|
||||
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
|
||||
return AWS_LL_FATAL;
|
||||
}
|
||||
|
||||
static void initialiseAwsLogger()
|
||||
{
|
||||
static std::once_flag initialised; /* aws_logger_set must only be called once */
|
||||
std::call_once(initialised, []() {
|
||||
static aws_logger_vtable nixAwsLoggerVtable = {
|
||||
.log = nixAwsLoggerLog,
|
||||
.get_log_level = nixAwsLoggerGetLevel,
|
||||
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
|
||||
.set_log_level = nullptr,
|
||||
};
|
||||
|
||||
static aws_logger nixAwsLogger = {
|
||||
.vtable = &nixAwsLoggerVtable,
|
||||
.allocator = nullptr,
|
||||
.p_impl = nullptr,
|
||||
};
|
||||
|
||||
aws_logger_set(&nixAwsLogger);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to wrap a C credentials provider in the C++ interface.
|
||||
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
|
||||
@@ -219,9 +119,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
|
||||
public:
|
||||
AwsCredentialProviderImpl()
|
||||
{
|
||||
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
|
||||
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
|
||||
initialiseAwsLogger();
|
||||
// Map Nix's verbosity to AWS CRT log level
|
||||
Aws::Crt::LogLevel logLevel;
|
||||
if (verbosity >= lvlVomit) {
|
||||
logLevel = Aws::Crt::LogLevel::Trace;
|
||||
} else if (verbosity >= lvlDebug) {
|
||||
logLevel = Aws::Crt::LogLevel::Debug;
|
||||
} else if (verbosity >= lvlChatty) {
|
||||
logLevel = Aws::Crt::LogLevel::Info;
|
||||
} else {
|
||||
logLevel = Aws::Crt::LogLevel::Warn;
|
||||
}
|
||||
apiHandle.InitializeLogging(logLevel, stderr);
|
||||
|
||||
// Create a shared TLS context for SSO (required for HTTPS connections)
|
||||
auto allocator = Aws::Crt::ApiAllocator();
|
||||
|
||||
@@ -71,14 +71,6 @@ void DerivationBuildingGoal::killChild()
|
||||
#endif
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::timedOut(Error && ex)
|
||||
{
|
||||
killChild();
|
||||
// We're not inside a coroutine, hence we can't use co_return here.
|
||||
// Thus we ignore the return value.
|
||||
[[maybe_unused]] Done _ = doneFailure({BuildResult::Failure::TimedOut, std::move(ex)});
|
||||
}
|
||||
|
||||
std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
|
||||
{
|
||||
std::string msg;
|
||||
@@ -443,7 +435,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
if (useHook) {
|
||||
buildResult.startTime = time(0); // inexact
|
||||
started();
|
||||
co_await Suspend{};
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
co_await processChildOutput(output->fd, output->data);
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
killChild();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
assert(hook);
|
||||
@@ -563,7 +568,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
{
|
||||
DerivationBuildingGoal & goal;
|
||||
|
||||
DerivationBuildingGoalCallbacks(DerivationBuildingGoal & goal)
|
||||
DerivationBuildingGoalCallbacks(
|
||||
DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
|
||||
: goal{goal}
|
||||
{
|
||||
}
|
||||
@@ -631,15 +637,15 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
|
||||
/* If we have to wait and retry (see below), then `builder` will
|
||||
already be created, so we don't need to create it again. */
|
||||
builder =
|
||||
externalBuilder
|
||||
? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP, std::make_unique<DerivationBuildingGoalCallbacks>(*this), std::move(params));
|
||||
builder = externalBuilder ? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params));
|
||||
}
|
||||
|
||||
if (auto builderOutOpt = builder->startBuild()) {
|
||||
@@ -663,7 +669,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
worker.childStarted(shared_from_this(), {builderOut}, true, true);
|
||||
|
||||
started();
|
||||
co_await Suspend{};
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
co_await processChildOutput(output->fd, output->data);
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
killChild();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
|
||||
trace("build done");
|
||||
|
||||
@@ -969,7 +988,7 @@ Path DerivationBuildingGoal::openLogFile()
|
||||
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
||||
|
||||
if (settings.compressLog)
|
||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
|
||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
|
||||
else
|
||||
logSink = logFileSink;
|
||||
|
||||
@@ -996,7 +1015,7 @@ bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
|
||||
#endif
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||
Goal::Co DerivationBuildingGoal::processChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
@@ -1004,14 +1023,11 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
killChild();
|
||||
// We're not inside a coroutine, hence we can't use co_return here.
|
||||
// Thus we ignore the return value.
|
||||
[[maybe_unused]] Done _ = doneFailure(BuildError(
|
||||
co_return doneFailure(BuildError(
|
||||
BuildResult::Failure::LogLimitExceeded,
|
||||
"%s killed after writing more than %d bytes of log output",
|
||||
getName(),
|
||||
settings.maxLogSize));
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto c : data)
|
||||
@@ -1064,13 +1080,7 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
|
||||
currentHookLine += c;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
worker.wakeUp(shared_from_this());
|
||||
co_return Return{};
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::flushLine()
|
||||
@@ -1175,6 +1185,11 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = std::move(builtOutputs),
|
||||
};
|
||||
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (status == BuildResult::Success::Built)
|
||||
@@ -1182,15 +1197,16 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status stat
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = std::move(builtOutputs),
|
||||
});
|
||||
return amDone(ecSuccess, std::nullopt);
|
||||
}
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
|
||||
{
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
};
|
||||
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (ex.status == BuildResult::Failure::TimedOut)
|
||||
@@ -1202,13 +1218,7 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
},
|
||||
std::move(ex));
|
||||
return amDone(ecFailed, {std::move(ex)});
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||
#include "nix/store/build/derivation-building-goal.hh"
|
||||
#include "nix/store/build/derivation-resolution-goal.hh"
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
@@ -100,9 +101,24 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
through substitutes. If that doesn't work, we'll build
|
||||
them. */
|
||||
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
|
||||
if (!checkResult)
|
||||
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
|
||||
else {
|
||||
if (!checkResult) {
|
||||
DrvOutput id{outputHash, wantedOutput};
|
||||
auto g = worker.makeDrvOutputSubstitutionGoal(id);
|
||||
waitees.insert(g);
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
if (nrFailed == 0) {
|
||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath)));
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed == 0)
|
||||
worker.store.registerDrvOutput({*g->outputInfo, id});
|
||||
else
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
}
|
||||
} else {
|
||||
auto * cap = getDerivationCA(*drv);
|
||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
|
||||
checkResult->first.outPath,
|
||||
@@ -210,11 +226,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
.outputName = wantedOutput,
|
||||
}};
|
||||
newRealisation.signatures.clear();
|
||||
if (!drv->type().isFixed()) {
|
||||
auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
|
||||
newRealisation.dependentRealisations =
|
||||
drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
|
||||
}
|
||||
worker.store.signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
}
|
||||
@@ -452,6 +463,20 @@ UnkeyedRealisation DerivationGoal::assertPathValidity()
|
||||
|
||||
Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, UnkeyedRealisation builtOutput)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
};
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
|
||||
if (status == BuildResult::Success::Built)
|
||||
@@ -459,24 +484,16 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
});
|
||||
return amDone(ecSuccess, std::nullopt);
|
||||
}
|
||||
|
||||
Goal::Done DerivationGoal::doneFailure(BuildError ex)
|
||||
{
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
};
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
|
||||
if (ex.status == BuildResult::Failure::TimedOut)
|
||||
@@ -488,13 +505,7 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
},
|
||||
std::move(ex));
|
||||
return amDone(ecFailed, {std::move(ex)});
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -98,13 +98,7 @@ Goal::Co DerivationTrampolineGoal::init()
|
||||
trace("outer load and build derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
co_return doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = BuildResult::Failure::DependencyFailed,
|
||||
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
|
||||
},
|
||||
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
|
||||
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
|
||||
}
|
||||
|
||||
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/util/callback.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -21,11 +19,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
trace("init");
|
||||
|
||||
/* If the derivation already exists, we’re done */
|
||||
if (worker.store.queryRealisation(id)) {
|
||||
if ((outputInfo = worker.store.queryRealisation(id))) {
|
||||
co_return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
||||
auto subs = worker.getSubstituters();
|
||||
|
||||
bool substituterFailed = false;
|
||||
|
||||
@@ -66,16 +64,19 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
true,
|
||||
false);
|
||||
|
||||
co_await Suspend{};
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (std::get_if<ChildOutput>(&event)) {
|
||||
// Doesn't process child output
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
break;
|
||||
} else if (std::get_if<TimedOut>(&event)) {
|
||||
unreachable();
|
||||
}
|
||||
}
|
||||
|
||||
worker.childTerminated(this);
|
||||
|
||||
/*
|
||||
* The realisation corresponding to the given output id.
|
||||
* Will be filled once we can get it.
|
||||
*/
|
||||
std::shared_ptr<const UnkeyedRealisation> outputInfo;
|
||||
|
||||
try {
|
||||
outputInfo = promise->get_future().get();
|
||||
} catch (std::exception & e) {
|
||||
@@ -86,45 +87,6 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
if (!outputInfo)
|
||||
continue;
|
||||
|
||||
bool failed = false;
|
||||
|
||||
Goals waitees;
|
||||
|
||||
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
|
||||
if (depId != id) {
|
||||
if (auto localOutputInfo = worker.store.queryRealisation(depId);
|
||||
localOutputInfo && localOutputInfo->outPath != depPath) {
|
||||
warn(
|
||||
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
|
||||
"Local: %s\n"
|
||||
"Remote: %s",
|
||||
sub->config.getHumanReadableURI(),
|
||||
depId.to_string(),
|
||||
worker.store.printStorePath(localOutputInfo->outPath),
|
||||
worker.store.printStorePath(depPath));
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
|
||||
}
|
||||
}
|
||||
|
||||
if (failed)
|
||||
continue;
|
||||
|
||||
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
|
||||
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
|
||||
}
|
||||
|
||||
worker.store.registerDrvOutput({*outputInfo, id});
|
||||
|
||||
trace("finished");
|
||||
co_return amDone(ecSuccess);
|
||||
}
|
||||
@@ -149,9 +111,4 @@ std::string DrvOutputSubstitutionGoal::key()
|
||||
return "a$" + std::string(id.to_string());
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -63,18 +63,12 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
|
||||
std::vector<KeyedBuildResult> results;
|
||||
results.reserve(state.size());
|
||||
|
||||
for (auto & [req, goalPtr] : state) {
|
||||
/* Goals that were never started or were cancelled have exitCode
|
||||
ecBusy and a default buildResult with empty errorMsg. Skip them
|
||||
to avoid reporting spurious failures with empty messages. */
|
||||
if (goalPtr->exitCode == Goal::ecBusy)
|
||||
continue;
|
||||
for (auto & [req, goalPtr] : state)
|
||||
results.emplace_back(
|
||||
KeyedBuildResult{
|
||||
goalPtr->buildResult,
|
||||
/* .path = */ req,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
+71
-13
@@ -4,8 +4,58 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
TimedOut::TimedOut(time_t maxDuration)
|
||||
: BuildError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration)
|
||||
, maxDuration(maxDuration)
|
||||
{
|
||||
}
|
||||
|
||||
using Co = nix::Goal::Co;
|
||||
using promise_type = nix::Goal::promise_type;
|
||||
using ChildEvents = decltype(promise_type::childEvents);
|
||||
|
||||
void ChildEvents::pushChildEvent(ChildOutput event)
|
||||
{
|
||||
if (childTimeout)
|
||||
return; // Already timed out, ignore
|
||||
childOutputs.push(std::move(event));
|
||||
}
|
||||
|
||||
void ChildEvents::pushChildEvent(ChildEOF event)
|
||||
{
|
||||
if (childTimeout)
|
||||
return; // Already timed out, ignore
|
||||
assert(!childEOF);
|
||||
childEOF = std::move(event);
|
||||
}
|
||||
|
||||
void ChildEvents::pushChildEvent(TimedOut event)
|
||||
{
|
||||
// Timeout is immediate - flush pending events
|
||||
childOutputs = {};
|
||||
childEOF.reset();
|
||||
childTimeout = std::move(event);
|
||||
}
|
||||
|
||||
bool ChildEvents::hasChildEvent() const
|
||||
{
|
||||
return !childOutputs.empty() || childEOF || childTimeout;
|
||||
}
|
||||
|
||||
Goal::ChildEvent ChildEvents::popChildEvent()
|
||||
{
|
||||
if (!childOutputs.empty()) {
|
||||
auto event = std::move(childOutputs.front());
|
||||
childOutputs.pop();
|
||||
return event;
|
||||
}
|
||||
if (childEOF)
|
||||
return *std::exchange(childEOF, std::nullopt);
|
||||
if (childTimeout)
|
||||
return *std::exchange(childTimeout, std::nullopt);
|
||||
unreachable();
|
||||
}
|
||||
|
||||
using handle_type = nix::Goal::handle_type;
|
||||
using Suspend = nix::Goal::Suspend;
|
||||
|
||||
@@ -133,19 +183,6 @@ Co Goal::await(Goals new_waitees)
|
||||
co_return Return{};
|
||||
}
|
||||
|
||||
Goal::Done Goal::doneSuccess(BuildResult::Success success)
|
||||
{
|
||||
buildResult.inner = std::move(success);
|
||||
return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
|
||||
{
|
||||
assert(result == ecFailed || result == ecNoSubstituters);
|
||||
buildResult.inner = std::move(failure);
|
||||
return amDone(result, std::move(ex));
|
||||
}
|
||||
|
||||
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
|
||||
{
|
||||
trace("done");
|
||||
@@ -219,6 +256,27 @@ void Goal::work()
|
||||
assert(top_co || exitCode != ecBusy);
|
||||
}
|
||||
|
||||
void Goal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
assert(top_co);
|
||||
top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}});
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
void Goal::handleEOF(Descriptor fd)
|
||||
{
|
||||
assert(top_co);
|
||||
top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd});
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
void Goal::timedOut(TimedOut && ex)
|
||||
{
|
||||
assert(top_co);
|
||||
top_co->handle.promise().childEvents.pushChildEvent(std::move(ex));
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
Goal::Co Goal::yield()
|
||||
{
|
||||
worker.wakeUp(shared_from_this());
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/store/nar-info.hh"
|
||||
#include "nix/util/finally.hh"
|
||||
@@ -29,21 +28,20 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
|
||||
|
||||
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
|
||||
{
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
});
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
};
|
||||
return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
|
||||
{
|
||||
debug(errorMsg);
|
||||
return Goal::doneFailure(
|
||||
result,
|
||||
BuildResult::Failure{
|
||||
.status = status,
|
||||
.errorMsg = std::move(errorMsg),
|
||||
});
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = status,
|
||||
.errorMsg = std::move(errorMsg),
|
||||
};
|
||||
return amDone(result);
|
||||
}
|
||||
|
||||
Goal::Co PathSubstitutionGoal::init()
|
||||
@@ -61,7 +59,7 @@ Goal::Co PathSubstitutionGoal::init()
|
||||
throw Error(
|
||||
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
|
||||
|
||||
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
||||
auto subs = worker.getSubstituters();
|
||||
|
||||
bool substituterFailed = false;
|
||||
std::optional<Error> lastStoresException = std::nullopt;
|
||||
@@ -259,7 +257,16 @@ Goal::Co PathSubstitutionGoal::tryToRun(
|
||||
true,
|
||||
false);
|
||||
|
||||
co_await Suspend{};
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (std::get_if<ChildOutput>(&event)) {
|
||||
// Substitution doesn't process child output
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
break;
|
||||
} else if (std::get_if<TimedOut>(&event)) {
|
||||
unreachable(); // Substitution doesn't use timeouts
|
||||
}
|
||||
}
|
||||
|
||||
trace("substitute finished");
|
||||
|
||||
@@ -311,11 +318,6 @@ Goal::Co PathSubstitutionGoal::tryToRun(
|
||||
co_return doneSuccess(BuildResult::Success::Substituted);
|
||||
}
|
||||
|
||||
void PathSubstitutionGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
||||
void PathSubstitutionGoal::cleanup()
|
||||
{
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "nix/store/local-store.hh"
|
||||
#include "nix/store/machines.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||
@@ -21,6 +22,7 @@ Worker::Worker(Store & store, Store & evalStore)
|
||||
, actSubstitutions(*logger, actCopyPaths)
|
||||
, store(store)
|
||||
, evalStore(evalStore)
|
||||
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
|
||||
{
|
||||
nrLocalBuilds = 0;
|
||||
nrSubstitutions = 0;
|
||||
@@ -479,14 +481,13 @@ void Worker::waitForInput()
|
||||
|
||||
if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts
|
||||
&& after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
|
||||
goal->timedOut(
|
||||
Error("%1% timed out after %2% seconds of silence", goal->getName(), settings.maxSilentTime));
|
||||
goal->timedOut(TimedOut(settings.maxSilentTime));
|
||||
}
|
||||
|
||||
else if (
|
||||
goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts
|
||||
&& after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) {
|
||||
goal->timedOut(Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout));
|
||||
goal->timedOut(TimedOut(settings.buildTimeout));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -852,10 +852,7 @@ static void performOp(
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
std::shared_ptr<const ValidPathInfo> info;
|
||||
logger->startWork();
|
||||
try {
|
||||
info = store->queryPathInfo(path);
|
||||
} catch (InvalidPath &) {
|
||||
}
|
||||
info = store->queryPathInfo(path);
|
||||
logger->stopWork();
|
||||
if (info) {
|
||||
conn.to << 1;
|
||||
|
||||
@@ -341,8 +341,12 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
|
||||
|
||||
if (parsed) {
|
||||
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
|
||||
if (!e || !e->is_object())
|
||||
if (!e)
|
||||
return ret;
|
||||
if (!e->is_object()) {
|
||||
warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring");
|
||||
return ret;
|
||||
}
|
||||
for (auto & [key, storePathsJson] : getObject(*e)) {
|
||||
StringSet ss;
|
||||
flatten(storePathsJson, ss);
|
||||
|
||||
+68
-103
@@ -30,8 +30,6 @@
|
||||
#include <thread>
|
||||
#include <regex>
|
||||
|
||||
using namespace std::string_literals;
|
||||
|
||||
namespace nix {
|
||||
|
||||
const unsigned int RETRY_TIME_MS_DEFAULT = 250;
|
||||
@@ -41,9 +39,27 @@ FileTransferSettings fileTransferSettings;
|
||||
|
||||
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
|
||||
|
||||
namespace {
|
||||
|
||||
using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>;
|
||||
using curlMulti = std::unique_ptr<::CURLM, decltype([](::CURLM * multi) { ::curl_multi_cleanup(multi); })>;
|
||||
|
||||
struct curlMultiError : Error
|
||||
{
|
||||
::CURLMcode code;
|
||||
|
||||
curlMultiError(::CURLMcode code)
|
||||
: Error{"unexpected curl multi error: %s", ::curl_multi_strerror(code)}
|
||||
{
|
||||
assert(code != CURLM_OK);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
CURLM * curlm = 0;
|
||||
curlMulti curlm;
|
||||
|
||||
std::random_device rd;
|
||||
std::mt19937 mt19937;
|
||||
@@ -59,8 +75,9 @@ struct curlFileTransfer : public FileTransfer
|
||||
CURL * req = 0;
|
||||
// buffer to accompany the `req` above
|
||||
char errbuf[CURL_ERROR_SIZE];
|
||||
bool active = false; // whether the handle has been added to the multi object
|
||||
bool paused = false; // whether the request has been paused previously
|
||||
bool active = false; // whether the handle has been added to the multi object
|
||||
bool paused = false; // whether the request has been paused previously
|
||||
bool enqueued = false; // whether the request has been added the incoming queue
|
||||
std::string statusMsg;
|
||||
|
||||
unsigned int attempt = 0;
|
||||
@@ -69,7 +86,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
has been reached. */
|
||||
std::chrono::steady_clock::time_point embargo;
|
||||
|
||||
struct curl_slist * requestHeaders = 0;
|
||||
curlSList requestHeaders;
|
||||
|
||||
std::string encoding;
|
||||
|
||||
@@ -92,6 +109,15 @@ struct curlFileTransfer : public FileTransfer
|
||||
return httpStatus;
|
||||
}
|
||||
|
||||
void appendHeaders(const std::string & header)
|
||||
{
|
||||
curlSList tmpSList = curlSList(::curl_slist_append(requestHeaders.get(), requireCString(header)));
|
||||
if (!tmpSList)
|
||||
throw std::bad_alloc();
|
||||
requestHeaders.release();
|
||||
requestHeaders = std::move(tmpSList);
|
||||
}
|
||||
|
||||
TransferItem(
|
||||
curlFileTransfer & fileTransfer,
|
||||
const FileTransferRequest & request,
|
||||
@@ -131,22 +157,13 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
result.urls.push_back(request.uri.to_string());
|
||||
|
||||
/* Don't set Accept-Encoding for S3 requests that use AWS SigV4 signing.
|
||||
curl's SigV4 implementation signs all headers including Accept-Encoding,
|
||||
but some S3-compatible services (like GCS) modify this header in transit,
|
||||
causing signature verification to fail.
|
||||
See https://github.com/NixOS/nix/issues/15019 */
|
||||
#if NIX_WITH_AWS_AUTH
|
||||
if (!request.awsSigV4Provider)
|
||||
#endif
|
||||
requestHeaders =
|
||||
curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
appendHeaders("Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
if (!request.expectedETag.empty())
|
||||
requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
|
||||
appendHeaders("If-None-Match: " + request.expectedETag);
|
||||
if (!request.mimeType.empty())
|
||||
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
|
||||
appendHeaders("Content-Type: " + request.mimeType);
|
||||
for (auto it = request.headers.begin(); it != request.headers.end(); ++it) {
|
||||
requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
|
||||
appendHeaders(fmt("%s: %s", it->first, it->second));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,13 +171,11 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
if (req) {
|
||||
if (active)
|
||||
curl_multi_remove_handle(fileTransfer.curlm, req);
|
||||
curl_multi_remove_handle(fileTransfer.curlm.get(), req);
|
||||
curl_easy_cleanup(req);
|
||||
}
|
||||
if (requestHeaders)
|
||||
curl_slist_free_all(requestHeaders);
|
||||
try {
|
||||
if (!done)
|
||||
if (!done && enqueued)
|
||||
fail(FileTransferError(
|
||||
Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri));
|
||||
} catch (...) {
|
||||
@@ -357,7 +372,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
|
||||
#if !defined(_WIN32)
|
||||
static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose)
|
||||
{
|
||||
unix::closeOnExec(curlfd);
|
||||
@@ -420,15 +435,11 @@ struct curlFileTransfer : public FileTransfer
|
||||
("curl/" LIBCURL_VERSION " Nix/" + nixVersion
|
||||
+ (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : ""))
|
||||
.c_str());
|
||||
#if LIBCURL_VERSION_NUM >= 0x072b00
|
||||
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
|
||||
#endif
|
||||
#if LIBCURL_VERSION_NUM >= 0x072f00
|
||||
if (fileTransferSettings.enableHttp2)
|
||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
|
||||
else
|
||||
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
|
||||
#endif
|
||||
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
|
||||
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
|
||||
@@ -438,7 +449,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
|
||||
curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
|
||||
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders.get());
|
||||
|
||||
if (settings.downloadSpeed.get() > 0)
|
||||
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
|
||||
@@ -450,11 +461,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
|
||||
if (request.data) {
|
||||
// Restart the source to ensure it's at the beginning.
|
||||
// This is necessary for retries, where the source was
|
||||
// already consumed by a previous attempt.
|
||||
request.data->source->restart();
|
||||
|
||||
if (request.method == HttpMethod::Post) {
|
||||
curl_easy_setopt(req, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
|
||||
@@ -473,18 +479,11 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (settings.caFile != "")
|
||||
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
|
||||
|
||||
#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
|
||||
#if !defined(_WIN32)
|
||||
curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
|
||||
#endif
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
|
||||
|
||||
// Enable TCP keep-alive so that idle connections in curl's reuse pool
|
||||
// are not silently dropped by NATs, firewalls, or load balancers.
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPALIVE, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPIDLE, 60L);
|
||||
curl_easy_setopt(req, CURLOPT_TCP_KEEPINTVL, 60L);
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
|
||||
|
||||
@@ -714,13 +713,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
Sync<State> state_;
|
||||
|
||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
||||
/* We can't use a std::condition_variable to wake up the curl
|
||||
thread, because it only monitors file descriptors. So use a
|
||||
pipe instead. */
|
||||
Pipe wakeupPipe;
|
||||
#endif
|
||||
|
||||
std::thread workerThread;
|
||||
|
||||
curlFileTransfer()
|
||||
@@ -729,43 +721,35 @@ struct curlFileTransfer : public FileTransfer
|
||||
static std::once_flag globalInit;
|
||||
std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
|
||||
|
||||
curlm = curl_multi_init();
|
||||
curlm = curlMulti(curl_multi_init());
|
||||
|
||||
#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0
|
||||
curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
#endif
|
||||
#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
|
||||
curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
|
||||
#endif
|
||||
|
||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
||||
wakeupPipe.create();
|
||||
fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
|
||||
#endif
|
||||
curl_multi_setopt(curlm.get(), CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
|
||||
|
||||
workerThread = std::thread([&]() { workerThreadEntry(); });
|
||||
}
|
||||
|
||||
~curlFileTransfer()
|
||||
{
|
||||
stopWorkerThread();
|
||||
|
||||
try {
|
||||
stopWorkerThread();
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
}
|
||||
workerThread.join();
|
||||
|
||||
if (curlm)
|
||||
curl_multi_cleanup(curlm);
|
||||
}
|
||||
|
||||
void stopWorkerThread()
|
||||
{
|
||||
/* Signal the worker thread to exit. */
|
||||
{
|
||||
auto state(state_.lock());
|
||||
state->quit();
|
||||
}
|
||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
||||
writeFull(wakeupPipe.writeSide.get(), " ", false);
|
||||
#endif
|
||||
state_.lock()->quit();
|
||||
wakeupMulti();
|
||||
}
|
||||
|
||||
void wakeupMulti()
|
||||
{
|
||||
if (auto ec = ::curl_multi_wakeup(curlm.get()))
|
||||
throw curlMultiError(ec);
|
||||
}
|
||||
|
||||
void workerThreadMain()
|
||||
@@ -795,32 +779,25 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
/* Let curl do its thing. */
|
||||
int running;
|
||||
CURLMcode mc = curl_multi_perform(curlm, &running);
|
||||
CURLMcode mc = curl_multi_perform(curlm.get(), &running);
|
||||
if (mc != CURLM_OK)
|
||||
throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
|
||||
|
||||
/* Set the promises of any finished requests. */
|
||||
CURLMsg * msg;
|
||||
int left;
|
||||
while ((msg = curl_multi_info_read(curlm, &left))) {
|
||||
while ((msg = curl_multi_info_read(curlm.get(), &left))) {
|
||||
if (msg->msg == CURLMSG_DONE) {
|
||||
auto i = items.find(msg->easy_handle);
|
||||
assert(i != items.end());
|
||||
i->second->finish(msg->data.result);
|
||||
curl_multi_remove_handle(curlm, i->second->req);
|
||||
curl_multi_remove_handle(curlm.get(), i->second->req);
|
||||
i->second->active = false;
|
||||
items.erase(i);
|
||||
}
|
||||
}
|
||||
|
||||
/* Wait for activity, including wakeup events. */
|
||||
int numfds = 0;
|
||||
struct curl_waitfd extraFDs[1];
|
||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
||||
extraFDs[0].fd = wakeupPipe.readSide.get();
|
||||
extraFDs[0].events = CURL_WAIT_POLLIN;
|
||||
extraFDs[0].revents = 0;
|
||||
#endif
|
||||
long maxSleepTimeMs = items.empty() ? 10000 : 100;
|
||||
auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point()
|
||||
? std::max(
|
||||
@@ -829,23 +806,14 @@ struct curlFileTransfer : public FileTransfer
|
||||
nextWakeup - std::chrono::steady_clock::now())
|
||||
.count())
|
||||
: maxSleepTimeMs;
|
||||
vomit("download thread waiting for %d ms", sleepTimeMs);
|
||||
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
|
||||
|
||||
int numfds = 0;
|
||||
mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds);
|
||||
if (mc != CURLM_OK)
|
||||
throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
|
||||
throw curlMultiError(mc);
|
||||
|
||||
nextWakeup = std::chrono::steady_clock::time_point();
|
||||
|
||||
/* Add new curl requests from the incoming requests queue,
|
||||
except for requests that are embargoed (waiting for a
|
||||
retry timeout to expire). */
|
||||
if (extraFDs[0].revents & CURL_WAIT_POLLIN) {
|
||||
char buf[1024];
|
||||
auto res = read(extraFDs[0].fd, buf, sizeof(buf));
|
||||
if (res == -1 && errno != EINTR)
|
||||
throw SysError("reading curl wakeup socket");
|
||||
}
|
||||
|
||||
std::vector<std::shared_ptr<TransferItem>> incoming;
|
||||
auto now = std::chrono::steady_clock::now();
|
||||
|
||||
@@ -868,7 +836,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
for (auto & item : incoming) {
|
||||
debug("starting %s of %s", item->request.noun(), item->request.uri);
|
||||
item->init();
|
||||
curl_multi_add_handle(curlm, item->req);
|
||||
curl_multi_add_handle(curlm.get(), item->req);
|
||||
item->active = true;
|
||||
items[item->req] = item;
|
||||
}
|
||||
@@ -918,11 +886,10 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (state->isQuitting())
|
||||
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
|
||||
state->incoming.push(item);
|
||||
item->enqueued = true; /* Now any exceptions should be reported via the callback. */
|
||||
}
|
||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
||||
writeFull(wakeupPipe.writeSide.get(), " ");
|
||||
#endif
|
||||
|
||||
wakeupMulti();
|
||||
return ItemHandle(static_cast<Item &>(*item));
|
||||
}
|
||||
|
||||
@@ -942,9 +909,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
auto state(state_.lock());
|
||||
state->unpause.push_back(std::move(item));
|
||||
#ifndef _WIN32 // TODO need graceful async exit support on Windows?
|
||||
writeFull(wakeupPipe.writeSide.get(), " ");
|
||||
#endif
|
||||
wakeupMulti();
|
||||
}
|
||||
|
||||
void unpauseTransfer(ItemHandle handle) override
|
||||
|
||||
@@ -189,22 +189,15 @@ struct ExternalBuilder
|
||||
std::vector<std::string> args;
|
||||
};
|
||||
|
||||
struct DerivationBuilderDeleter
|
||||
{
|
||||
void operator()(DerivationBuilder * builder) noexcept;
|
||||
};
|
||||
|
||||
using DerivationBuilderUnique = std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter>;
|
||||
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
DerivationBuilderUnique makeDerivationBuilder(
|
||||
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
|
||||
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params);
|
||||
|
||||
/**
|
||||
* @param handler Must be chosen such that it supports the given
|
||||
* derivation.
|
||||
*/
|
||||
DerivationBuilderUnique makeExternalDerivationBuilder(
|
||||
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
|
||||
LocalStore & store,
|
||||
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
|
||||
DerivationBuilderParams params,
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
#include "nix/store/parsed-derivations.hh"
|
||||
#include "nix/store/derivation-options.hh"
|
||||
#include "nix/store/build/derivation-building-misc.hh"
|
||||
#include "nix/store/build/derivation-builder.hh"
|
||||
#include "nix/store/outputs-spec.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/store/pathlocks.hh"
|
||||
@@ -90,7 +89,7 @@ private:
|
||||
*/
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
|
||||
DerivationBuilderUnique builder;
|
||||
std::unique_ptr<DerivationBuilder> builder;
|
||||
#endif
|
||||
|
||||
BuildMode buildMode;
|
||||
@@ -101,8 +100,6 @@ private:
|
||||
|
||||
std::map<ActivityId, Activity> builderActivities;
|
||||
|
||||
void timedOut(Error && ex) override;
|
||||
|
||||
std::string key() override;
|
||||
|
||||
/**
|
||||
@@ -130,10 +127,10 @@ private:
|
||||
bool isReadDesc(Descriptor fd);
|
||||
|
||||
/**
|
||||
* Callback used by the worker to write to the log.
|
||||
* Process output from a child process.
|
||||
*/
|
||||
void handleChildOutput(Descriptor fd, std::string_view data) override;
|
||||
void handleEOF(Descriptor fd) override;
|
||||
Co processChildOutput(Descriptor fd, std::string_view data);
|
||||
|
||||
void flushLine();
|
||||
|
||||
/**
|
||||
|
||||
@@ -52,11 +52,6 @@ struct DerivationGoal : public Goal
|
||||
bool storeDerivation);
|
||||
~DerivationGoal() = default;
|
||||
|
||||
void timedOut(Error && ex) override
|
||||
{
|
||||
unreachable();
|
||||
};
|
||||
|
||||
std::string key() override;
|
||||
|
||||
JobCategory jobCategory() const override
|
||||
|
||||
@@ -43,8 +43,6 @@ struct DerivationResolutionGoal : public Goal
|
||||
*/
|
||||
std::unique_ptr<std::pair<StorePath, BasicDerivation>> resolvedDrv;
|
||||
|
||||
void timedOut(Error && ex) override {}
|
||||
|
||||
private:
|
||||
|
||||
/**
|
||||
|
||||
@@ -109,8 +109,6 @@ struct DerivationTrampolineGoal : public Goal
|
||||
|
||||
virtual ~DerivationTrampolineGoal();
|
||||
|
||||
void timedOut(Error && ex) override {}
|
||||
|
||||
std::string key() override;
|
||||
|
||||
JobCategory jobCategory() const override
|
||||
|
||||
@@ -14,11 +14,14 @@ namespace nix {
|
||||
class Worker;
|
||||
|
||||
/**
|
||||
* Substitution of a derivation output.
|
||||
* This is done in three steps:
|
||||
* 1. Fetch the output info from a substituter
|
||||
* 2. Substitute the corresponding output path
|
||||
* 3. Register the output info
|
||||
* Fetch a `Realisation` (drv ⨯ output name -> output path) from a
|
||||
* substituter.
|
||||
*
|
||||
* If the output store object itself should also be substituted, that is
|
||||
* the responsibility of the caller to do so.
|
||||
*
|
||||
* @todo rename this `BuidlTraceEntryGoal`, which will make sense
|
||||
* especially once `Realisation` is renamed to `BuildTraceEntry`.
|
||||
*/
|
||||
class DrvOutputSubstitutionGoal : public Goal
|
||||
{
|
||||
@@ -31,17 +34,16 @@ class DrvOutputSubstitutionGoal : public Goal
|
||||
public:
|
||||
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
|
||||
|
||||
/**
|
||||
* The realisation corresponding to the given output id.
|
||||
* Will be filled once we can get it.
|
||||
*/
|
||||
std::shared_ptr<const UnkeyedRealisation> outputInfo;
|
||||
|
||||
Co init();
|
||||
|
||||
void timedOut(Error && ex) override
|
||||
{
|
||||
unreachable();
|
||||
};
|
||||
|
||||
std::string key() override;
|
||||
|
||||
void handleEOF(Descriptor fd) override;
|
||||
|
||||
JobCategory jobCategory() const override
|
||||
{
|
||||
return JobCategory::Substitution;
|
||||
|
||||
@@ -5,9 +5,18 @@
|
||||
#include "nix/store/build-result.hh"
|
||||
|
||||
#include <coroutine>
|
||||
#include <queue>
|
||||
#include <variant>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct TimedOut : BuildError
|
||||
{
|
||||
time_t maxDuration;
|
||||
|
||||
TimedOut(time_t maxDuration);
|
||||
};
|
||||
|
||||
/**
|
||||
* Forward definition.
|
||||
*/
|
||||
@@ -138,6 +147,29 @@ public:
|
||||
friend Goal;
|
||||
};
|
||||
|
||||
/**
|
||||
* Event types for child process communication, delivered via coroutines.
|
||||
*/
|
||||
struct ChildOutput
|
||||
{
|
||||
Descriptor fd;
|
||||
std::string data;
|
||||
};
|
||||
|
||||
struct ChildEOF
|
||||
{
|
||||
Descriptor fd;
|
||||
};
|
||||
|
||||
using ChildEvent = std::variant<ChildOutput, ChildEOF, TimedOut>;
|
||||
|
||||
/**
|
||||
* Tag type for `co_await`-ing child events.
|
||||
* Returns a `ChildEvent` when resumed.
|
||||
*/
|
||||
struct WaitForChildEvent
|
||||
{};
|
||||
|
||||
// forward declaration of promise_type, see below
|
||||
struct promise_type;
|
||||
|
||||
@@ -276,6 +308,28 @@ public:
|
||||
*/
|
||||
bool alive = true;
|
||||
|
||||
class
|
||||
{
|
||||
/**
|
||||
* Structured queue of child events:
|
||||
* - outputs: stream of data from child
|
||||
* - eof: optional end-of-stream marker
|
||||
* - timeout: optional timeout that flushes/overrides other events
|
||||
*/
|
||||
std::queue<ChildOutput> childOutputs;
|
||||
std::optional<ChildEOF> childEOF;
|
||||
std::optional<TimedOut> childTimeout;
|
||||
|
||||
public:
|
||||
|
||||
void pushChildEvent(ChildOutput event);
|
||||
void pushChildEvent(ChildEOF event);
|
||||
void pushChildEvent(TimedOut event);
|
||||
bool hasChildEvent() const;
|
||||
ChildEvent popChildEvent();
|
||||
|
||||
} childEvents;
|
||||
|
||||
/**
|
||||
* The awaiter used by @ref final_suspend.
|
||||
*/
|
||||
@@ -369,13 +423,66 @@ public:
|
||||
return static_cast<Co &&>(co);
|
||||
}
|
||||
|
||||
/**
|
||||
* Awaiter for @ref Suspend. Always suspends, but asserts
|
||||
* there are no pending child events (those should be
|
||||
* consumed first via @ref WaitForChildEvent).
|
||||
*/
|
||||
struct SuspendAwaiter
|
||||
{
|
||||
promise_type & promise;
|
||||
|
||||
bool await_ready()
|
||||
{
|
||||
assert(!promise.childEvents.hasChildEvent());
|
||||
return false;
|
||||
}
|
||||
|
||||
void await_suspend(handle_type) {}
|
||||
|
||||
void await_resume() {}
|
||||
};
|
||||
|
||||
/**
|
||||
* Allows awaiting a @ref Suspend.
|
||||
* Always suspends.
|
||||
*/
|
||||
std::suspend_always await_transform(Suspend)
|
||||
SuspendAwaiter await_transform(Suspend)
|
||||
{
|
||||
return {};
|
||||
return SuspendAwaiter{*this};
|
||||
};
|
||||
|
||||
/**
|
||||
* Awaiter for child events. Suspends and returns the
|
||||
* pending child event when resumed.
|
||||
*/
|
||||
struct ChildEventAwaiter
|
||||
{
|
||||
handle_type handle;
|
||||
|
||||
bool await_ready()
|
||||
{
|
||||
return handle && handle.promise().childEvents.hasChildEvent();
|
||||
}
|
||||
|
||||
void await_suspend(handle_type h)
|
||||
{
|
||||
handle = h;
|
||||
}
|
||||
|
||||
ChildEvent await_resume()
|
||||
{
|
||||
assert(handle);
|
||||
return handle.promise().childEvents.popChildEvent();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Allows awaiting child events (output, EOF, timeout).
|
||||
*/
|
||||
ChildEventAwaiter await_transform(WaitForChildEvent)
|
||||
{
|
||||
return ChildEventAwaiter{handle_type::from_promise(*this)};
|
||||
};
|
||||
};
|
||||
|
||||
@@ -393,28 +500,9 @@ protected:
|
||||
* Signals that the goal is done.
|
||||
* `co_return` the result. If you're not inside a coroutine, you can ignore
|
||||
* the return value safely.
|
||||
*
|
||||
* Prefer using `doneSuccess` or `doneFailure` instead, which ensure
|
||||
* `buildResult` is set correctly.
|
||||
*/
|
||||
Done amDone(ExitCode result, std::optional<Error> ex = {});
|
||||
|
||||
/**
|
||||
* Signals successful completion of the goal.
|
||||
* Sets `buildResult` and calls `amDone`.
|
||||
*/
|
||||
Done doneSuccess(BuildResult::Success success);
|
||||
|
||||
/**
|
||||
* Signals failed completion of the goal.
|
||||
* Sets `buildResult` and calls `amDone`.
|
||||
*
|
||||
* @param result The exit code (ecFailed or ecNoSubstituters)
|
||||
* @param failure The failure details including status and error message
|
||||
* @param ex Optional exception to store/log
|
||||
*/
|
||||
Done doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex = {});
|
||||
|
||||
public:
|
||||
virtual void cleanup() {}
|
||||
|
||||
@@ -451,15 +539,23 @@ public:
|
||||
|
||||
void work();
|
||||
|
||||
virtual void handleChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
unreachable();
|
||||
}
|
||||
/**
|
||||
* Called by the worker when data is received from a child process.
|
||||
* Stores the event and resumes the coroutine.
|
||||
*/
|
||||
void handleChildOutput(Descriptor fd, std::string_view data);
|
||||
|
||||
virtual void handleEOF(Descriptor fd)
|
||||
{
|
||||
unreachable();
|
||||
}
|
||||
/**
|
||||
* Called by the worker when EOF is received from a child process.
|
||||
* Stores the event and resumes the coroutine.
|
||||
*/
|
||||
void handleEOF(Descriptor fd);
|
||||
|
||||
/**
|
||||
* Called by the worker when a build times out.
|
||||
* Stores the event and resumes the coroutine.
|
||||
*/
|
||||
void timedOut(TimedOut && ex);
|
||||
|
||||
void trace(std::string_view s);
|
||||
|
||||
@@ -468,13 +564,6 @@ public:
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Callback in case of a timeout. It should wake up its waiters,
|
||||
* get rid of any running child processes that are being monitored
|
||||
* by the worker (important!), etc.
|
||||
*/
|
||||
virtual void timedOut(Error && ex) = 0;
|
||||
|
||||
/**
|
||||
* Used for comparisons. The order matters a bit for scheduling. We
|
||||
* want:
|
||||
|
||||
@@ -53,11 +53,6 @@ public:
|
||||
std::optional<ContentAddress> ca = std::nullopt);
|
||||
~PathSubstitutionGoal();
|
||||
|
||||
void timedOut(Error && ex) override
|
||||
{
|
||||
unreachable();
|
||||
};
|
||||
|
||||
std::string key() override
|
||||
{
|
||||
return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
|
||||
@@ -72,12 +67,6 @@ public:
|
||||
StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
|
||||
Co finished();
|
||||
|
||||
/**
|
||||
* Callback used by the worker to write to the log.
|
||||
*/
|
||||
void handleChildOutput(Descriptor fd, std::string_view data) override {};
|
||||
void handleEOF(Descriptor fd) override;
|
||||
|
||||
/* Called by destructor, can't be overridden */
|
||||
void cleanup() override final;
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "nix/store/realisation.hh"
|
||||
#include "nix/util/muxable-pipe.hh"
|
||||
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <thread>
|
||||
|
||||
@@ -171,6 +172,14 @@ public:
|
||||
Store & store;
|
||||
Store & evalStore;
|
||||
|
||||
/**
|
||||
* Function to get the substituters to use for path substitution.
|
||||
*
|
||||
* Defaults to `getDefaultSubstituters`. This allows tests to
|
||||
* inject custom substituters.
|
||||
*/
|
||||
std::function<std::list<ref<Store>>()> getSubstituters;
|
||||
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
#endif
|
||||
|
||||
@@ -420,7 +420,7 @@ private:
|
||||
|
||||
uint64_t queryValidPathId(State & state, const StorePath & path);
|
||||
|
||||
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
|
||||
uint64_t addValidPath(State & state, const ValidPathInfo & info);
|
||||
|
||||
void invalidatePath(State & state, const StorePath & path);
|
||||
|
||||
|
||||
@@ -33,6 +33,22 @@ private:
|
||||
public:
|
||||
PathLocks();
|
||||
PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
|
||||
|
||||
PathLocks(PathLocks && other) noexcept
|
||||
: fds(std::exchange(other.fds, {}))
|
||||
, deletePaths(other.deletePaths)
|
||||
{
|
||||
}
|
||||
|
||||
PathLocks & operator=(PathLocks && other) noexcept
|
||||
{
|
||||
fds = std::exchange(other.fds, {});
|
||||
deletePaths = other.deletePaths;
|
||||
return *this;
|
||||
}
|
||||
|
||||
PathLocks(const PathLocks &) = delete;
|
||||
PathLocks & operator=(const PathLocks &) = delete;
|
||||
bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
|
||||
~PathLocks();
|
||||
void unlock();
|
||||
@@ -45,6 +61,10 @@ struct FdLock
|
||||
bool acquired = false;
|
||||
|
||||
FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
|
||||
FdLock(const FdLock &) = delete;
|
||||
FdLock & operator=(const FdLock &) = delete;
|
||||
FdLock(FdLock &&) = delete;
|
||||
FdLock & operator=(FdLock &&) = delete;
|
||||
|
||||
~FdLock()
|
||||
{
|
||||
|
||||
@@ -56,14 +56,6 @@ struct UnkeyedRealisation
|
||||
|
||||
StringSet signatures;
|
||||
|
||||
/**
|
||||
* The realisations that are required for the current one to be valid.
|
||||
*
|
||||
* When importing this realisation, the store will first check that all its
|
||||
* dependencies exist, and map to the correct output path
|
||||
*/
|
||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
||||
|
||||
std::string fingerprint(const DrvOutput & key) const;
|
||||
|
||||
void sign(const DrvOutput & key, const Signer &);
|
||||
@@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation
|
||||
|
||||
bool isCompatibleWith(const UnkeyedRealisation & other) const;
|
||||
|
||||
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
|
||||
|
||||
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
|
||||
|
||||
bool operator==(const Realisation &) const = default;
|
||||
auto operator<=>(const Realisation &) const = default;
|
||||
};
|
||||
@@ -154,10 +142,6 @@ struct RealisedPath
|
||||
*/
|
||||
const StorePath & path() const &;
|
||||
|
||||
void closure(Store & store, Set & ret) const;
|
||||
static void closure(Store & store, const Set & startPaths, Set & ret);
|
||||
Set closure(Store & store) const;
|
||||
|
||||
bool operator==(const RealisedPath &) const = default;
|
||||
auto operator<=>(const RealisedPath &) const = default;
|
||||
};
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
#include "nix/store/config.hh"
|
||||
#include "nix/store/http-binary-cache-store.hh"
|
||||
#include "nix/store/s3-url.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -53,22 +52,13 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
"endpoint",
|
||||
R"(
|
||||
The S3 endpoint to use. When empty (default), uses AWS S3 with
|
||||
region-specific endpoints. For S3-compatible services such as
|
||||
MinIO, set this to your service's endpoint.
|
||||
)"};
|
||||
region-specific endpoints (e.g., s3.us-east-1.amazonaws.com).
|
||||
For S3-compatible services such as MinIO, set this to your service's endpoint.
|
||||
|
||||
Setting<S3AddressingStyle> addressingStyle{
|
||||
this,
|
||||
S3AddressingStyle::Auto,
|
||||
"addressing-style",
|
||||
R"(
|
||||
The S3 addressing style to use. `auto` (default) uses
|
||||
virtual-hosted-style for standard AWS endpoints and path-style
|
||||
for custom endpoints; bucket names containing dots automatically
|
||||
fall back to path-style to avoid TLS certificate errors. `path`
|
||||
forces path-style addressing (deprecated by AWS). `virtual`
|
||||
forces virtual-hosted-style addressing (bucket names must not
|
||||
contain dots).
|
||||
> **Note**
|
||||
>
|
||||
> Custom endpoints must support HTTPS and use path-based
|
||||
> addressing instead of virtual host based addressing.
|
||||
)"};
|
||||
|
||||
const Setting<bool> multipartUpload{
|
||||
@@ -127,7 +117,7 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
|
||||
* Set of settings that are part of the S3 URI itself.
|
||||
* These are needed for region specification and other S3-specific settings.
|
||||
*/
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint, &addressingStyle};
|
||||
const std::set<const AbstractSetting *> s3UriSettings = {&profile, ®ion, &scheme, &endpoint};
|
||||
|
||||
static const std::string name()
|
||||
{
|
||||
|
||||
@@ -1,41 +1,16 @@
|
||||
#pragma once
|
||||
///@file
|
||||
#include "nix/store/config.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/url.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* S3 addressing style for bucket access.
|
||||
* - Auto: virtual-hosted-style for standard AWS endpoints, path-style for custom endpoints.
|
||||
* - Path: always use path-style (bucket in URL path).
|
||||
* - Virtual: always use virtual-hosted-style (bucket as hostname prefix; bucket name must not contain dots).
|
||||
*/
|
||||
enum class S3AddressingStyle {
|
||||
Auto,
|
||||
Path,
|
||||
Virtual,
|
||||
};
|
||||
|
||||
MakeError(InvalidS3AddressingStyle, Error);
|
||||
|
||||
S3AddressingStyle parseS3AddressingStyle(std::string_view style);
|
||||
std::string_view showS3AddressingStyle(S3AddressingStyle style);
|
||||
|
||||
template<>
|
||||
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const;
|
||||
|
||||
template<>
|
||||
std::string BaseSetting<S3AddressingStyle>::to_string() const;
|
||||
|
||||
/**
|
||||
* Parsed S3 URL.
|
||||
*/
|
||||
@@ -52,7 +27,6 @@ struct ParsedS3URL
|
||||
std::optional<std::string> region;
|
||||
std::optional<std::string> scheme;
|
||||
std::optional<std::string> versionId;
|
||||
std::optional<S3AddressingStyle> addressingStyle;
|
||||
/**
|
||||
* The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
|
||||
* or an authority (so an IP address or a registered name).
|
||||
@@ -72,8 +46,7 @@ struct ParsedS3URL
|
||||
static ParsedS3URL parse(const ParsedURL & uri);
|
||||
|
||||
/**
|
||||
* Convert this ParsedS3URL to an HTTP(S) ParsedURL for use with curl's AWS SigV4 authentication.
|
||||
* The scheme defaults to HTTPS but respects the 'scheme' setting and custom endpoint schemes.
|
||||
* Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication
|
||||
*/
|
||||
ParsedURL toHttpsUrl() const;
|
||||
|
||||
|
||||
@@ -1007,9 +1007,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
|
||||
|
||||
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
||||
|
||||
std::map<DrvOutput, StorePath>
|
||||
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<TrustedFlag> : std::true_type
|
||||
{};
|
||||
|
||||
@@ -110,8 +110,6 @@ struct LocalStore::State::Stmts
|
||||
SQLiteStmt QueryAllRealisedOutputs;
|
||||
SQLiteStmt QueryPathFromHashPart;
|
||||
SQLiteStmt QueryValidPaths;
|
||||
SQLiteStmt QueryRealisationReferences;
|
||||
SQLiteStmt AddRealisationReference;
|
||||
};
|
||||
|
||||
LocalStore::LocalStore(ref<const Config> config)
|
||||
@@ -390,21 +388,6 @@ LocalStore::LocalStore(ref<const Config> config)
|
||||
where drvPath = ?
|
||||
;
|
||||
)");
|
||||
state->stmts->QueryRealisationReferences.create(
|
||||
state->db,
|
||||
R"(
|
||||
select drvPath, outputName from Realisations
|
||||
join RealisationsRefs on realisationReference = Realisations.id
|
||||
where referrer = ?;
|
||||
)");
|
||||
state->stmts->AddRealisationReference.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert or replace into RealisationsRefs (referrer, realisationReference)
|
||||
values (
|
||||
(select id from Realisations where drvPath = ? and outputName = ?),
|
||||
(select id from Realisations where drvPath = ? and outputName = ?));
|
||||
)");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -654,25 +637,6 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
concatStringsSep(" ", info.signatures))
|
||||
.exec();
|
||||
}
|
||||
for (auto & [outputId, depPath] : info.dependentRealisations) {
|
||||
auto localRealisation = queryRealisationCore_(*state, outputId);
|
||||
if (!localRealisation)
|
||||
throw Error(
|
||||
"unable to register the derivation '%s' as it "
|
||||
"depends on the non existent '%s'",
|
||||
info.id.to_string(),
|
||||
outputId.to_string());
|
||||
if (localRealisation->second.outPath != depPath)
|
||||
throw Error(
|
||||
"unable to register the derivation '%s' as it "
|
||||
"depends on a realisation of '%s' that doesn’t"
|
||||
"match what we have locally",
|
||||
info.id.to_string(),
|
||||
outputId.to_string());
|
||||
state->stmts->AddRealisationReference
|
||||
.use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
|
||||
.exec();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -683,7 +647,7 @@ void LocalStore::cacheDrvOutputMapping(
|
||||
[&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); });
|
||||
}
|
||||
|
||||
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs)
|
||||
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info)
|
||||
{
|
||||
if (info.ca.has_value() && !info.isContentAddressed(*this))
|
||||
throw Error(
|
||||
@@ -704,17 +668,16 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo
|
||||
efficiently query whether a path is an output of some
|
||||
derivation. */
|
||||
if (info.path.isDerivation()) {
|
||||
auto drv = readInvalidDerivation(info.path);
|
||||
auto parsedDrv = readInvalidDerivation(info.path);
|
||||
|
||||
/* Verify that the output paths in the derivation are correct
|
||||
(i.e., follow the scheme for computing output paths from
|
||||
derivations). Note that if this throws an error, then the
|
||||
DB transaction is rolled back, so the path validity
|
||||
registration above is undone. */
|
||||
if (checkOutputs)
|
||||
drv.checkInvariants(*this, info.path);
|
||||
parsedDrv.checkInvariants(*this, info.path);
|
||||
|
||||
for (auto & i : drv.outputsAndOptPaths(*this)) {
|
||||
for (auto & i : parsedDrv.outputsAndOptPaths(*this)) {
|
||||
/* Floating CA derivations have indeterminate output paths until
|
||||
they are built, so don't register anything in that case */
|
||||
if (i.second.second)
|
||||
@@ -965,7 +928,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||
if (isValidPath_(*state, i.path))
|
||||
updatePathInfo(*state, i);
|
||||
else
|
||||
addValidPath(*state, i, false);
|
||||
addValidPath(*state, i);
|
||||
paths.insert(i.path);
|
||||
}
|
||||
|
||||
@@ -975,15 +938,6 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
||||
}
|
||||
|
||||
/* Check that the derivation outputs are correct. We can't do
|
||||
this in addValidPath() above, because the references might
|
||||
not be valid yet. */
|
||||
for (auto & [_, i] : infos)
|
||||
if (i.path.isDerivation()) {
|
||||
// FIXME: inefficient; we already loaded the derivation in addValidPath().
|
||||
readInvalidDerivation(i.path).checkInvariants(*this, i.path);
|
||||
}
|
||||
|
||||
/* Do a topological sort of the paths. This will throw an
|
||||
error if a cycle is detected and roll back the
|
||||
transaction. Cycles can only occur when a derivation
|
||||
@@ -1606,21 +1560,6 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
|
||||
return std::nullopt;
|
||||
auto [realisationDbId, res] = *maybeCore;
|
||||
|
||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
||||
auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
|
||||
while (useRealisationRefs.next()) {
|
||||
auto depId = DrvOutput{
|
||||
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
|
||||
useRealisationRefs.getStr(1),
|
||||
};
|
||||
auto dependentRealisation = queryRealisationCore_(state, depId);
|
||||
assert(dependentRealisation); // Enforced by the db schema
|
||||
auto outputPath = dependentRealisation->second.outPath;
|
||||
dependentRealisations.insert({depId, outputPath});
|
||||
}
|
||||
|
||||
res.dependentRealisations = dependentRealisations;
|
||||
|
||||
return {res};
|
||||
}
|
||||
|
||||
|
||||
+26
-22
@@ -39,28 +39,32 @@ deps_public_maybe_subproject = [
|
||||
]
|
||||
subdir('nix-meson-build-support/subprojects')
|
||||
|
||||
run_command(
|
||||
'ln',
|
||||
'-s',
|
||||
meson.project_build_root() / '__nothing_link_target',
|
||||
meson.project_build_root() / '__nothing_symlink',
|
||||
# native doesn't allow dangling symlinks, which the tests require
|
||||
env : {'MSYS' : 'winsymlinks:lnk'},
|
||||
check : true,
|
||||
)
|
||||
can_link_symlink = run_command(
|
||||
'ln',
|
||||
meson.project_build_root() / '__nothing_symlink',
|
||||
meson.project_build_root() / '__nothing_hardlink',
|
||||
check : false,
|
||||
).returncode() == 0
|
||||
run_command(
|
||||
'rm',
|
||||
'-f',
|
||||
meson.project_build_root() / '__nothing_symlink',
|
||||
meson.project_build_root() / '__nothing_hardlink',
|
||||
check : true,
|
||||
)
|
||||
can_link_symlink = false
|
||||
native_ln = find_program('ln', required : false, native : true)
|
||||
if native_ln.found()
|
||||
run_command(
|
||||
native_ln,
|
||||
'-s',
|
||||
meson.project_build_root() / '__nothing_link_target',
|
||||
meson.project_build_root() / '__nothing_symlink',
|
||||
# native doesn't allow dangling symlinks, which the tests require
|
||||
env : {'MSYS' : 'winsymlinks:lnk'},
|
||||
check : true,
|
||||
)
|
||||
can_link_symlink = run_command(
|
||||
native_ln,
|
||||
meson.project_build_root() / '__nothing_symlink',
|
||||
meson.project_build_root() / '__nothing_hardlink',
|
||||
check : false,
|
||||
).returncode() == 0
|
||||
run_command(
|
||||
'rm',
|
||||
'-f',
|
||||
meson.project_build_root() / '__nothing_symlink',
|
||||
meson.project_build_root() / '__nothing_hardlink',
|
||||
check : true,
|
||||
)
|
||||
endif
|
||||
summary('can hardlink to symlink', can_link_symlink, bool_yn : true)
|
||||
configdata_priv.set('CAN_LINK_SYMLINK', can_link_symlink.to_int())
|
||||
|
||||
|
||||
+9
-82
@@ -23,9 +23,9 @@ void Store::computeFSClosure(
|
||||
bool includeOutputs,
|
||||
bool includeDerivers)
|
||||
{
|
||||
std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
|
||||
std::function<asio::awaitable<StorePathSet>(const StorePath & path)> queryDeps;
|
||||
if (flipDirection)
|
||||
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
|
||||
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
|
||||
StorePathSet res;
|
||||
StorePathSet referrers;
|
||||
queryReferrers(path, referrers);
|
||||
@@ -41,12 +41,14 @@ void Store::computeFSClosure(
|
||||
for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
|
||||
if (maybeOutPath && isValidPath(*maybeOutPath))
|
||||
res.insert(*maybeOutPath);
|
||||
return res;
|
||||
co_return res;
|
||||
};
|
||||
else
|
||||
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
|
||||
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
|
||||
StorePathSet res;
|
||||
auto info = fut.get();
|
||||
auto info = co_await callbackToAwaitable<ref<const ValidPathInfo>>(
|
||||
[this, path](Callback<ref<const ValidPathInfo>> cb) { queryPathInfo(path, std::move(cb)); });
|
||||
|
||||
for (auto & ref : info->references)
|
||||
if (ref != path)
|
||||
res.insert(ref);
|
||||
@@ -58,25 +60,9 @@ void Store::computeFSClosure(
|
||||
|
||||
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
|
||||
res.insert(*info->deriver);
|
||||
return res;
|
||||
co_return res;
|
||||
};
|
||||
|
||||
computeClosure<StorePath>(
|
||||
startPaths,
|
||||
paths_,
|
||||
[&](const StorePath & path, std::function<void(std::promise<std::set<StorePath>> &)> processEdges) {
|
||||
std::promise<std::set<StorePath>> promise;
|
||||
std::function<void(std::future<ref<const ValidPathInfo>>)> getDependencies =
|
||||
[&](std::future<ref<const ValidPathInfo>> fut) {
|
||||
try {
|
||||
promise.set_value(queryDeps(path, fut));
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
};
|
||||
queryPathInfo(path, getDependencies);
|
||||
processEdges(promise);
|
||||
});
|
||||
computeClosure<StorePath>(startPaths, paths_, queryDeps);
|
||||
}
|
||||
|
||||
void Store::computeFSClosure(
|
||||
@@ -334,65 +320,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
|
||||
result);
|
||||
}
|
||||
|
||||
std::map<DrvOutput, StorePath>
|
||||
drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
|
||||
{
|
||||
std::map<DrvOutput, StorePath> res;
|
||||
|
||||
for (const auto & input : inputRealisations) {
|
||||
if (pathReferences.count(input.outPath)) {
|
||||
res.insert({input.id, input.outPath});
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::map<DrvOutput, StorePath>
|
||||
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
|
||||
{
|
||||
auto & evalStore = evalStore_ ? *evalStore_ : store;
|
||||
|
||||
std::set<Realisation> inputRealisations;
|
||||
|
||||
auto accumRealisations = [&](this auto & self,
|
||||
const StorePath & inputDrv,
|
||||
const DerivedPathMap<StringSet>::ChildNode & inputNode) -> void {
|
||||
if (!inputNode.value.empty()) {
|
||||
auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
|
||||
for (const auto & outputName : inputNode.value) {
|
||||
auto outputHash = get(outputHashes, outputName);
|
||||
if (!outputHash)
|
||||
throw Error(
|
||||
"output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
|
||||
DrvOutput key{*outputHash, outputName};
|
||||
auto thisRealisation = store.queryRealisation(key);
|
||||
if (!thisRealisation)
|
||||
throw Error(
|
||||
"output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
|
||||
inputRealisations.insert({*thisRealisation, std::move(key)});
|
||||
}
|
||||
}
|
||||
if (!inputNode.value.empty()) {
|
||||
auto d = makeConstantStorePathRef(inputDrv);
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap) {
|
||||
SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
|
||||
self(
|
||||
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
|
||||
resolveDerivedPath(store, next, evalStore_),
|
||||
childNode);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
||||
accumRealisations(inputDrv, inputNode);
|
||||
|
||||
auto info = store.queryPathInfo(outputPath);
|
||||
|
||||
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
|
||||
}
|
||||
|
||||
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
|
||||
{
|
||||
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#include "nix/store/realisation.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/util/closure.hh"
|
||||
#include "nix/util/signature/local-keys.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const
|
||||
return strHash() + "!" + outputName;
|
||||
}
|
||||
|
||||
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
|
||||
{
|
||||
std::set<Realisation> res;
|
||||
Realisation::closure(store, startOutputs, res);
|
||||
return res;
|
||||
}
|
||||
|
||||
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
|
||||
{
|
||||
auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
|
||||
std::set<Realisation> res;
|
||||
for (auto & [currentDep, _] : current.dependentRealisations) {
|
||||
if (auto currentRealisation = store.queryRealisation(currentDep))
|
||||
res.insert({*currentRealisation, currentDep});
|
||||
else
|
||||
throw Error("Unrealised derivation '%s'", currentDep.to_string());
|
||||
}
|
||||
return res;
|
||||
};
|
||||
|
||||
computeClosure<Realisation>(
|
||||
startOutputs,
|
||||
res,
|
||||
[&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
|
||||
std::promise<std::set<Realisation>> promise;
|
||||
try {
|
||||
auto res = getDeps(current);
|
||||
promise.set_value(res);
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
return processEdges(promise);
|
||||
});
|
||||
}
|
||||
|
||||
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
|
||||
{
|
||||
nlohmann::json serialized = Realisation{*this, key};
|
||||
@@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const &
|
||||
|
||||
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
|
||||
{
|
||||
if (outPath == other.outPath) {
|
||||
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
|
||||
warn(
|
||||
"Encountered a realisation for '%s' with an empty set of "
|
||||
"dependencies. This is likely an artifact from an older Nix. "
|
||||
"I’ll try to fix the realisation if I can",
|
||||
id.to_string());
|
||||
return true;
|
||||
} else if (dependentRealisations == other.dependentRealisations) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret)
|
||||
{
|
||||
// FIXME: This only builds the store-path closure, not the real realisation
|
||||
// closure
|
||||
StorePathSet initialStorePaths, pathsClosure;
|
||||
for (auto & path : startPaths)
|
||||
initialStorePaths.insert(path.path());
|
||||
store.computeFSClosure(initialStorePaths, pathsClosure);
|
||||
ret.insert(startPaths.begin(), startPaths.end());
|
||||
ret.insert(pathsClosure.begin(), pathsClosure.end());
|
||||
}
|
||||
|
||||
void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
|
||||
{
|
||||
RealisedPath::closure(store, {*this}, ret);
|
||||
}
|
||||
|
||||
RealisedPath::Set RealisedPath::closure(Store & store) const
|
||||
{
|
||||
RealisedPath::Set ret;
|
||||
closure(store, ret);
|
||||
return ret;
|
||||
return outPath == other.outPath;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -162,27 +90,19 @@ UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & js
|
||||
if (auto signaturesOpt = optionalValueAt(json, "signatures"))
|
||||
signatures = *signaturesOpt;
|
||||
|
||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
||||
if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations"))
|
||||
for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
|
||||
dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
|
||||
|
||||
return UnkeyedRealisation{
|
||||
.outPath = valueAt(json, "outPath"),
|
||||
.signatures = signatures,
|
||||
.dependentRealisations = dependentRealisations,
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
|
||||
{
|
||||
auto jsonDependentRealisations = nlohmann::json::object();
|
||||
for (auto & [depId, depOutPath] : r.dependentRealisations)
|
||||
jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
|
||||
json = {
|
||||
{"outPath", r.outPath},
|
||||
{"signatures", r.signatures},
|
||||
{"dependentRealisations", jsonDependentRealisations},
|
||||
// back-compat
|
||||
{"dependentRealisations", json::object()},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -292,7 +292,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
|
||||
next->computeFSClosure(newPaths, closure);
|
||||
for (auto & path : closure)
|
||||
goal.addDependency(path);
|
||||
for (auto & real : Realisation::closure(*next, newRealisations))
|
||||
for (auto & real : newRealisations)
|
||||
goal.addedDrvOutputs.insert(real.id);
|
||||
|
||||
return results;
|
||||
|
||||
+9
-137
@@ -1,14 +1,8 @@
|
||||
#include "nix/store/s3-url.hh"
|
||||
#include "nix/util/abstract-setting-to-json.hh"
|
||||
#include "nix/util/config-impl.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/logging.hh"
|
||||
#include "nix/util/json-impls.hh"
|
||||
#include "nix/util/split.hh"
|
||||
#include "nix/util/strings-inline.hh"
|
||||
|
||||
#include <atomic>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <ranges>
|
||||
#include <string_view>
|
||||
|
||||
@@ -16,30 +10,6 @@ using namespace std::string_view_literals;
|
||||
|
||||
namespace nix {
|
||||
|
||||
S3AddressingStyle parseS3AddressingStyle(std::string_view style)
|
||||
{
|
||||
if (style == "auto")
|
||||
return S3AddressingStyle::Auto;
|
||||
if (style == "path")
|
||||
return S3AddressingStyle::Path;
|
||||
if (style == "virtual")
|
||||
return S3AddressingStyle::Virtual;
|
||||
throw InvalidS3AddressingStyle("unknown S3 addressing style '%s', expected 'auto', 'path', or 'virtual'", style);
|
||||
}
|
||||
|
||||
std::string_view showS3AddressingStyle(S3AddressingStyle style)
|
||||
{
|
||||
switch (style) {
|
||||
case S3AddressingStyle::Auto:
|
||||
return "auto";
|
||||
case S3AddressingStyle::Path:
|
||||
return "path";
|
||||
case S3AddressingStyle::Virtual:
|
||||
return "virtual";
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed)
|
||||
try {
|
||||
if (parsed.scheme != "s3"sv)
|
||||
@@ -79,9 +49,6 @@ try {
|
||||
.region = getOptionalParam("region"),
|
||||
.scheme = getOptionalParam("scheme"),
|
||||
.versionId = getOptionalParam("versionId"),
|
||||
.addressingStyle = getOptionalParam("addressing-style").transform([](const std::string & s) {
|
||||
return parseS3AddressingStyle(s);
|
||||
}),
|
||||
.endpoint = [&]() -> decltype(ParsedS3URL::endpoint) {
|
||||
if (!endpoint)
|
||||
return std::monostate();
|
||||
@@ -98,9 +65,6 @@ try {
|
||||
} catch (BadURL & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
} catch (InvalidS3AddressingStyle & e) {
|
||||
e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string());
|
||||
throw;
|
||||
}
|
||||
|
||||
ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
@@ -116,95 +80,41 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
queryParams["versionId"] = *versionId;
|
||||
}
|
||||
|
||||
auto style = addressingStyle.value_or(S3AddressingStyle::Auto);
|
||||
|
||||
// Virtual-hosted-style prepends the bucket name to the hostname, so bucket
|
||||
// names containing dots produce multi-level subdomains (e.g.
|
||||
// my.bucket.s3.amazonaws.com) that break TLS wildcard certificate validation.
|
||||
// In auto mode, fall back to path-style; only error on explicit virtual.
|
||||
auto hasDottedBucket = bucket.find('.') != std::string::npos;
|
||||
auto useVirtualForEndpoint = [&](bool defaultVirtual) {
|
||||
auto useVirtual = defaultVirtual ? style != S3AddressingStyle::Path : style == S3AddressingStyle::Virtual;
|
||||
if (useVirtual && hasDottedBucket) {
|
||||
if (style == S3AddressingStyle::Virtual)
|
||||
throw Error(
|
||||
"bucket name '%s' contains a dot, which is incompatible with "
|
||||
"virtual-hosted-style addressing (causes TLS certificate errors); "
|
||||
"use 'addressing-style=path' or 'addressing-style=auto' instead",
|
||||
bucket);
|
||||
static std::atomic<bool> warnedDottedBucket{false};
|
||||
warnOnce(
|
||||
warnedDottedBucket,
|
||||
"bucket name '%s' contains a dot; falling back to path-style addressing "
|
||||
"(virtual-hosted-style requires non-dotted bucket names for TLS certificate validity); "
|
||||
"set 'addressing-style=path' to silence this warning",
|
||||
bucket);
|
||||
return false;
|
||||
}
|
||||
return useVirtual;
|
||||
};
|
||||
|
||||
// Handle endpoint configuration using std::visit
|
||||
return std::visit(
|
||||
overloaded{
|
||||
[&](const std::monostate &) {
|
||||
// No custom endpoint: use virtual-hosted-style by default (auto),
|
||||
// path-style when explicitly requested or for dotted bucket names.
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ true);
|
||||
// No custom endpoint, use standard AWS S3 endpoint
|
||||
std::vector<std::string> path{""};
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority =
|
||||
ParsedURL::Authority{
|
||||
.host = useVirtual ? bucket + ".s3." + regionStr + ".amazonaws.com"
|
||||
: "s3." + regionStr + ".amazonaws.com"},
|
||||
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL::Authority & auth) {
|
||||
// Custom endpoint authority: use path-style by default (auto),
|
||||
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
|
||||
if (useVirtual && auth.host.empty())
|
||||
throw Error(
|
||||
"cannot use virtual-hosted-style addressing with endpoint '%s' "
|
||||
"because it has no hostname; use 'addressing-style=path' instead",
|
||||
auth.to_string());
|
||||
// Endpoint is just an authority (hostname/port)
|
||||
std::vector<std::string> path{""};
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = std::string{schemeStr},
|
||||
.authority =
|
||||
useVirtual ? ParsedURL::Authority{.host = bucket + "." + auth.host, .port = auth.port} : auth,
|
||||
.authority = auth,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
},
|
||||
[&](const ParsedURL & endpointUrl) {
|
||||
// Full endpoint URL: use path-style by default (auto),
|
||||
// virtual-hosted-style only when explicitly requested (not for dotted buckets).
|
||||
auto useVirtual = useVirtualForEndpoint(/* defaultVirtual = */ false);
|
||||
if (useVirtual && (!endpointUrl.authority || endpointUrl.authority->host.empty()))
|
||||
throw Error(
|
||||
"cannot use virtual-hosted-style addressing with endpoint '%s' "
|
||||
"because it has no authority (hostname)",
|
||||
endpointUrl.to_string());
|
||||
// Endpoint is already a ParsedURL (e.g., http://server:9000)
|
||||
auto path = endpointUrl.path;
|
||||
if (!useVirtual)
|
||||
path.push_back(bucket);
|
||||
path.push_back(bucket);
|
||||
path.insert(path.end(), key.begin(), key.end());
|
||||
return ParsedURL{
|
||||
.scheme = endpointUrl.scheme,
|
||||
.authority = useVirtual ? std::optional{ParsedURL::Authority{
|
||||
.host = bucket + "." + endpointUrl.authority->host,
|
||||
.port = endpointUrl.authority->port}}
|
||||
: endpointUrl.authority,
|
||||
.authority = endpointUrl.authority,
|
||||
.path = std::move(path),
|
||||
.query = std::move(queryParams),
|
||||
};
|
||||
@@ -213,42 +123,4 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
|
||||
endpoint);
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json & j, const S3AddressingStyle & e)
|
||||
{
|
||||
j = std::string{showS3AddressingStyle(e)};
|
||||
}
|
||||
|
||||
void from_json(const nlohmann::json & j, S3AddressingStyle & e)
|
||||
{
|
||||
e = parseS3AddressingStyle(j.get<std::string>());
|
||||
}
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<S3AddressingStyle> : std::true_type
|
||||
{};
|
||||
|
||||
template<>
|
||||
S3AddressingStyle BaseSetting<S3AddressingStyle>::parse(const std::string & str) const
|
||||
{
|
||||
try {
|
||||
return parseS3AddressingStyle(str);
|
||||
} catch (InvalidS3AddressingStyle &) {
|
||||
throw UsageError("option '%s' has invalid value '%s', expected 'auto', 'path', or 'virtual'", name, str);
|
||||
}
|
||||
}
|
||||
|
||||
template<>
|
||||
std::string BaseSetting<S3AddressingStyle>::to_string() const
|
||||
{
|
||||
return std::string{showS3AddressingStyle(value)};
|
||||
}
|
||||
|
||||
template<>
|
||||
struct BaseSetting<S3AddressingStyle>::trait
|
||||
{
|
||||
static constexpr bool appendable = false;
|
||||
};
|
||||
|
||||
template class BaseSetting<S3AddressingStyle>;
|
||||
|
||||
} // namespace nix
|
||||
+18
-24
@@ -454,8 +454,6 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
|
||||
.downloadSize = narInfo ? narInfo->fileSize : 0,
|
||||
.narSize = info->narSize,
|
||||
});
|
||||
|
||||
break; /* We are done. */
|
||||
} catch (InvalidPath &) {
|
||||
} catch (SubstituterDisabled &) {
|
||||
} catch (Error & e) {
|
||||
@@ -917,36 +915,21 @@ std::map<StorePath, StorePath> copyPaths(
|
||||
SubstituteFlag substitute)
|
||||
{
|
||||
StorePathSet storePaths;
|
||||
std::set<Realisation> toplevelRealisations;
|
||||
std::vector<const Realisation *> realisations;
|
||||
for (auto & path : paths) {
|
||||
storePaths.insert(path.path());
|
||||
if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
|
||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||
toplevelRealisations.insert(*realisation);
|
||||
realisations.push_back(realisation);
|
||||
}
|
||||
}
|
||||
|
||||
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
|
||||
|
||||
try {
|
||||
// Copy the realisation closure
|
||||
processGraph<Realisation>(
|
||||
Realisation::closure(srcStore, toplevelRealisations),
|
||||
[&](const Realisation & current) -> std::set<Realisation> {
|
||||
std::set<Realisation> children;
|
||||
for (const auto & [drvOutput, _] : current.dependentRealisations) {
|
||||
auto currentChild = srcStore.queryRealisation(drvOutput);
|
||||
if (!currentChild)
|
||||
throw Error(
|
||||
"incomplete realisation closure: '%s' is a "
|
||||
"dependency of '%s' but isn't registered",
|
||||
drvOutput.to_string(),
|
||||
current.id.to_string());
|
||||
children.insert({*currentChild, drvOutput});
|
||||
}
|
||||
return children;
|
||||
},
|
||||
[&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
|
||||
// Copy the realisations. TODO batch this
|
||||
for (const auto * realisation : realisations)
|
||||
dstStore.registerDrvOutput(*realisation, checkSigs);
|
||||
} catch (MissingExperimentalFeature & e) {
|
||||
// Don't fail if the remote doesn't support CA derivations is it might
|
||||
// not be within our control to change that, and we might still want
|
||||
@@ -1057,8 +1040,19 @@ void copyClosure(
|
||||
if (&srcStore == &dstStore)
|
||||
return;
|
||||
|
||||
RealisedPath::Set closure;
|
||||
RealisedPath::closure(srcStore, paths, closure);
|
||||
StorePathSet closure0;
|
||||
for (auto & path : paths) {
|
||||
if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
|
||||
closure0.insert(opaquePath->path);
|
||||
}
|
||||
}
|
||||
|
||||
StorePathSet closure1;
|
||||
srcStore.computeFSClosure(closure0, closure1);
|
||||
|
||||
RealisedPath::Set closure = paths;
|
||||
for (auto && path : closure1)
|
||||
closure.insert({std::move(path)});
|
||||
|
||||
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
||||
}
|
||||
|
||||
@@ -98,13 +98,10 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup code to run when destroying any DerivationBuilderImpl implementation.
|
||||
*/
|
||||
void cleanupOnDestruction() noexcept
|
||||
~DerivationBuilderImpl()
|
||||
{
|
||||
/* Careful: we should never ever throw an exception from a
|
||||
noexcept function. */
|
||||
destructor. */
|
||||
try {
|
||||
killChild();
|
||||
} catch (...) {
|
||||
@@ -681,17 +678,17 @@ static void handleChildException(bool sendException)
|
||||
}
|
||||
}
|
||||
|
||||
static void checkNotWorldWritable(std::filesystem::path path)
|
||||
static bool checkNotWorldWritable(std::filesystem::path path)
|
||||
{
|
||||
while (true) {
|
||||
auto st = lstat(path);
|
||||
if (st.st_mode & S_IWOTH)
|
||||
throw Error("Path %s is world-writable or a symlink. That's not allowed for security.", path);
|
||||
return false;
|
||||
if (path == path.parent_path())
|
||||
break;
|
||||
path = path.parent_path();
|
||||
}
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<Descriptor> DerivationBuilderImpl::startBuild()
|
||||
@@ -713,8 +710,9 @@ std::optional<Descriptor> DerivationBuilderImpl::startBuild()
|
||||
|
||||
createDirs(buildDir);
|
||||
|
||||
if (buildUser)
|
||||
checkNotWorldWritable(buildDir);
|
||||
if (buildUser && !checkNotWorldWritable(buildDir))
|
||||
throw Error(
|
||||
"Path %s or a parent directory is world-writable or a symlink. That's not allowed for security.", buildDir);
|
||||
|
||||
/* Create a temporary directory where the build will take
|
||||
place. */
|
||||
@@ -1965,20 +1963,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
|
||||
|
||||
namespace nix {
|
||||
|
||||
void DerivationBuilderDeleter::operator()(DerivationBuilder * builder) noexcept
|
||||
{
|
||||
if (!builder) /* Idempotent and handles nullptr as any deleter must. */
|
||||
return;
|
||||
|
||||
if (auto builderImpl = dynamic_cast<DerivationBuilderImpl *>(builder))
|
||||
/* Note that this might call into virtual functions, which we can't do in a destructor of
|
||||
the DerivationBuilderImpl itself. */
|
||||
builderImpl->cleanupOnDestruction();
|
||||
|
||||
delete builder;
|
||||
}
|
||||
|
||||
std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuilder(
|
||||
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
|
||||
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
|
||||
{
|
||||
bool useSandbox = false;
|
||||
@@ -2029,19 +2014,17 @@ std::unique_ptr<DerivationBuilder, DerivationBuilderDeleter> makeDerivationBuild
|
||||
throw Error("feature 'uid-range' is only supported in sandboxed builds");
|
||||
|
||||
#ifdef __APPLE__
|
||||
return DerivationBuilderUnique(
|
||||
new DarwinDerivationBuilder(store, std::move(miscMethods), std::move(params), useSandbox));
|
||||
return std::make_unique<DarwinDerivationBuilder>(store, std::move(miscMethods), std::move(params), useSandbox);
|
||||
#elif defined(__linux__)
|
||||
if (useSandbox)
|
||||
return DerivationBuilderUnique(
|
||||
new ChrootLinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
|
||||
return std::make_unique<ChrootLinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
|
||||
|
||||
return DerivationBuilderUnique(new LinuxDerivationBuilder(store, std::move(miscMethods), std::move(params)));
|
||||
return std::make_unique<LinuxDerivationBuilder>(store, std::move(miscMethods), std::move(params));
|
||||
#else
|
||||
if (useSandbox)
|
||||
throw Error("sandboxing builds is not supported on this platform");
|
||||
|
||||
return DerivationBuilderUnique(new DerivationBuilderImpl(store, std::move(miscMethods), std::move(params)));
|
||||
return std::make_unique<DerivationBuilderImpl>(store, std::move(miscMethods), std::move(params));
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -106,14 +106,13 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl
|
||||
}
|
||||
};
|
||||
|
||||
DerivationBuilderUnique makeExternalDerivationBuilder(
|
||||
std::unique_ptr<DerivationBuilder> makeExternalDerivationBuilder(
|
||||
LocalStore & store,
|
||||
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
|
||||
DerivationBuilderParams params,
|
||||
const ExternalBuilder & handler)
|
||||
{
|
||||
return DerivationBuilderUnique(
|
||||
new ExternalDerivationBuilder(store, std::move(miscMethods), std::move(params), handler));
|
||||
return std::make_unique<ExternalDerivationBuilder>(store, std::move(miscMethods), std::move(params), handler);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -16,7 +16,6 @@ R""(
|
||||
|
||||
; Allow DNS lookups.
|
||||
(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))
|
||||
(allow mach-lookup (global-name "com.apple.SystemConfiguration.DNSConfiguration"))
|
||||
|
||||
; Allow access to trustd.
|
||||
(allow mach-lookup (global-name "com.apple.trustd"))
|
||||
|
||||
@@ -4,20 +4,11 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "nix/util/types.hh"
|
||||
#include "nix/util/environment-variables.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/tests/test-data.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* The path to the unit test data directory. See the contributing guide
|
||||
* in the manual for further details.
|
||||
*/
|
||||
static inline std::filesystem::path getUnitTestData()
|
||||
{
|
||||
return getEnv("_NIX_TEST_UNIT_DATA").value();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether we should update "golden masters" instead of running tests
|
||||
* against them. See the contributing guide in the manual for further
|
||||
|
||||
@@ -83,6 +83,31 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Specialization for when we need to do "JSON -> `ref<T>`" in one
|
||||
* direction, but "`const T &` -> JSON" in the other direction.
|
||||
*/
|
||||
template<typename T>
|
||||
void checkpointJson(CharacterizationTest & test, PathView testStem, const ref<T> & got)
|
||||
{
|
||||
using namespace nlohmann;
|
||||
|
||||
auto file = test.goldenMaster(Path{testStem} + ".json");
|
||||
|
||||
json gotJson = static_cast<json>(*got);
|
||||
|
||||
if (testAccept()) {
|
||||
std::filesystem::create_directories(file.parent_path());
|
||||
writeFile(file, gotJson.dump(2) + "\n");
|
||||
ADD_FAILURE() << "Updating golden master " << file;
|
||||
} else {
|
||||
json expectedJson = json::parse(readFile(file));
|
||||
ASSERT_EQ(gotJson, expectedJson);
|
||||
ref<T> expected = adl_serializer<ref<T>>::from_json(expectedJson);
|
||||
ASSERT_EQ(*got, *expected);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mixin class for writing characterization tests for `nlohmann::json`
|
||||
* conversions for a given type.
|
||||
|
||||
@@ -10,4 +10,5 @@ headers = files(
|
||||
'json-characterization.hh',
|
||||
'nix_api_util.hh',
|
||||
'string_callback.hh',
|
||||
'test-data.hh',
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <filesystem>
|
||||
#include "nix/util/environment-variables.hh"
|
||||
#include "nix/util/error.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* The path to the unit test data directory. See the contributing guide
|
||||
* in the manual for further details.
|
||||
*/
|
||||
static inline std::filesystem::path getUnitTestData()
|
||||
{
|
||||
auto data = getEnv("_NIX_TEST_UNIT_DATA");
|
||||
if (!data)
|
||||
throw Error(
|
||||
"_NIX_TEST_UNIT_DATA environment variable is not set. "
|
||||
"Recommendation: use meson, example: 'meson test -C build --gdb'");
|
||||
return std::filesystem::path(*data);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -15,32 +15,4 @@ TEST(alignUp, notAPowerOf2)
|
||||
ASSERT_DEATH({ alignUp(1u, 42); }, "alignment must be a power of 2");
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
class alignUpOverflowTest : public ::testing::Test
|
||||
{};
|
||||
|
||||
using UnsignedTypes = ::testing::Types<uint8_t, uint16_t, uint32_t, uint64_t>;
|
||||
TYPED_TEST_SUITE(alignUpOverflowTest, UnsignedTypes);
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, lastSafeValue)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_EQ(alignUp<TypeParam>(max - 15, 16), (max - 15) & ~TypeParam{15});
|
||||
ASSERT_NO_THROW(alignUp<TypeParam>(max - 15, 16));
|
||||
}
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, overflowThrows)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_THROW(alignUp<TypeParam>(max - 14, 16), Error);
|
||||
ASSERT_THROW(alignUp<TypeParam>(max, 16), Error);
|
||||
ASSERT_THROW(alignUp<TypeParam>(max, 2), Error);
|
||||
}
|
||||
|
||||
TYPED_TEST(alignUpOverflowTest, alignmentOneNeverOverflows)
|
||||
{
|
||||
constexpr auto max = std::numeric_limits<TypeParam>::max();
|
||||
ASSERT_EQ(alignUp<TypeParam>(max, 1), max);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -20,10 +20,8 @@ TEST(closure, correctClosure)
|
||||
set<string> aClosure;
|
||||
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
|
||||
computeClosure<string>(
|
||||
{"A"}, aClosure, [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
|
||||
promise<set<string>> promisedNodes;
|
||||
promisedNodes.set_value(testGraph[currentNode]);
|
||||
processEdges(promisedNodes);
|
||||
{"A"}, aClosure, [&](const std::string & currentNode) -> asio::awaitable<std::set<std::string>> {
|
||||
co_return testGraph[currentNode];
|
||||
});
|
||||
|
||||
ASSERT_EQ(aClosure, expectedClosure);
|
||||
@@ -37,31 +35,7 @@ TEST(closure, properlyHandlesDirectExceptions)
|
||||
set<string> aClosure;
|
||||
EXPECT_THROW(
|
||||
computeClosure<string>(
|
||||
{"A"},
|
||||
aClosure,
|
||||
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) { throw TestExn(); }),
|
||||
TestExn);
|
||||
}
|
||||
|
||||
TEST(closure, properlyHandlesExceptionsInPromise)
|
||||
{
|
||||
struct TestExn
|
||||
{};
|
||||
|
||||
set<string> aClosure;
|
||||
EXPECT_THROW(
|
||||
computeClosure<string>(
|
||||
{"A"},
|
||||
aClosure,
|
||||
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
|
||||
promise<set<string>> promise;
|
||||
try {
|
||||
throw TestExn();
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
processEdges(promise);
|
||||
}),
|
||||
{"A"}, aClosure, [&](const std::string &) -> asio::awaitable<std::set<std::string>> { throw TestExn(); }),
|
||||
TestExn);
|
||||
}
|
||||
|
||||
|
||||
@@ -388,14 +388,13 @@ TEST(openFileEnsureBeneathNoSymlinks, works)
|
||||
TEST(createAnonymousTempFile, works)
|
||||
{
|
||||
auto fd = createAnonymousTempFile();
|
||||
auto fd_ = fromDescriptorReadOnly(fd.get());
|
||||
writeFull(fd.get(), "test");
|
||||
lseek(fd_, 0, SEEK_SET);
|
||||
lseek(fd.get(), 0, SEEK_SET);
|
||||
FdSource source{fd.get()};
|
||||
EXPECT_EQ(source.drain(), "test");
|
||||
lseek(fd_, 0, SEEK_END);
|
||||
lseek(fd.get(), 0, SEEK_END);
|
||||
writeFull(fd.get(), "test");
|
||||
lseek(fd_, 0, SEEK_SET);
|
||||
lseek(fd.get(), 0, SEEK_SET);
|
||||
EXPECT_EQ(source.drain(), "testtest");
|
||||
}
|
||||
|
||||
@@ -406,9 +405,8 @@ TEST(createAnonymousTempFile, works)
|
||||
TEST(FdSource, restartWorks)
|
||||
{
|
||||
auto fd = createAnonymousTempFile();
|
||||
auto fd_ = fromDescriptorReadOnly(fd.get());
|
||||
writeFull(fd.get(), "hello world");
|
||||
lseek(fd_, 0, SEEK_SET);
|
||||
lseek(fd.get(), 0, SEEK_SET);
|
||||
FdSource source{fd.get()};
|
||||
EXPECT_EQ(source.drain(), "hello world");
|
||||
source.restart();
|
||||
@@ -416,4 +414,11 @@ TEST(FdSource, restartWorks)
|
||||
EXPECT_EQ(source.drain(), "");
|
||||
}
|
||||
|
||||
TEST(createTempDir, works)
|
||||
{
|
||||
auto tmpDir = createTempDir();
|
||||
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
|
||||
ASSERT_TRUE(std::filesystem::is_directory(tmpDir));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -33,6 +33,9 @@ deps_private += rapidcheck
|
||||
gtest = dependency('gtest', main : true)
|
||||
deps_private += gtest
|
||||
|
||||
gmock = dependency('gmock')
|
||||
deps_private += gmock
|
||||
|
||||
configdata = configuration_data()
|
||||
configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
|
||||
|
||||
@@ -72,6 +75,7 @@ sources = files(
|
||||
'position.cc',
|
||||
'processes.cc',
|
||||
'sort.cc',
|
||||
'source-accessor.cc',
|
||||
'spawn.cc',
|
||||
'strings.cc',
|
||||
'suggestions.cc',
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
#include "nix/util/fs-sink.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/processes.hh"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <gmock/gmock.h>
|
||||
#include <rapidcheck/gtest.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
MATCHER_P2(HasContents, path, expected, "")
|
||||
{
|
||||
auto stat = arg->maybeLstat(path);
|
||||
if (!stat) {
|
||||
*result_listener << arg->showPath(path) << " does not exist";
|
||||
return false;
|
||||
}
|
||||
if (stat->type != SourceAccessor::tRegular) {
|
||||
*result_listener << arg->showPath(path) << " is not a regular file";
|
||||
return false;
|
||||
}
|
||||
auto actual = arg->readFile(path);
|
||||
if (actual != expected) {
|
||||
*result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
MATCHER_P2(HasSymlink, path, target, "")
|
||||
{
|
||||
auto stat = arg->maybeLstat(path);
|
||||
if (!stat) {
|
||||
*result_listener << arg->showPath(path) << " does not exist";
|
||||
return false;
|
||||
}
|
||||
if (stat->type != SourceAccessor::tSymlink) {
|
||||
*result_listener << arg->showPath(path) << " is not a symlink";
|
||||
return false;
|
||||
}
|
||||
auto actual = arg->readLink(path);
|
||||
if (actual != target) {
|
||||
*result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
MATCHER_P2(HasDirectory, path, dirents, "")
|
||||
{
|
||||
auto stat = arg->maybeLstat(path);
|
||||
if (!stat) {
|
||||
*result_listener << arg->showPath(path) << " does not exist";
|
||||
return false;
|
||||
}
|
||||
if (stat->type != SourceAccessor::tDirectory) {
|
||||
*result_listener << arg->showPath(path) << " is not a directory";
|
||||
return false;
|
||||
}
|
||||
auto actual = arg->readDirectory(path);
|
||||
std::set<std::string> actualKeys, expectedKeys(dirents.begin(), dirents.end());
|
||||
for (auto & [k, _] : actual)
|
||||
actualKeys.insert(k);
|
||||
if (actualKeys != expectedKeys) {
|
||||
*result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
class FSSourceAccessorTest : public ::testing::Test
|
||||
{
|
||||
protected:
|
||||
std::filesystem::path tmpDir;
|
||||
std::unique_ptr<nix::AutoDelete> delTmpDir;
|
||||
|
||||
void SetUp() override
|
||||
{
|
||||
tmpDir = nix::createTempDir();
|
||||
delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
|
||||
}
|
||||
|
||||
void TearDown() override
|
||||
{
|
||||
delTmpDir.reset();
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(FSSourceAccessorTest, works)
|
||||
{
|
||||
{
|
||||
RestoreSink sink(false);
|
||||
sink.dstPath = tmpDir;
|
||||
#ifndef _WIN32
|
||||
sink.dirFd = openDirectory(tmpDir);
|
||||
#endif
|
||||
sink.createDirectory(CanonPath("subdir"));
|
||||
sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); });
|
||||
sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); });
|
||||
sink.createSymlink(CanonPath("rootlink"), "target");
|
||||
sink.createDirectory(CanonPath("a"));
|
||||
sink.createSymlink(CanonPath("a/dirlink"), "../subdir");
|
||||
}
|
||||
|
||||
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), HasContents(CanonPath::root, "content1"));
|
||||
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), HasSymlink(CanonPath::root, "target"));
|
||||
EXPECT_THAT(
|
||||
makeFSSourceAccessor(tmpDir),
|
||||
HasDirectory(CanonPath::root, std::set<std::string>{"file1", "subdir", "rootlink", "a"}));
|
||||
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "subdir"), HasDirectory(CanonPath::root, std::set<std::string>{"file2"}));
|
||||
|
||||
{
|
||||
auto accessor = makeFSSourceAccessor(tmpDir);
|
||||
EXPECT_THAT(accessor, HasContents(CanonPath("file1"), "content1"));
|
||||
EXPECT_THAT(accessor, HasContents(CanonPath("subdir/file2"), "content2"));
|
||||
|
||||
EXPECT_TRUE(accessor->pathExists(CanonPath("file1")));
|
||||
EXPECT_FALSE(accessor->pathExists(CanonPath("nonexistent")));
|
||||
|
||||
EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||
EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||
EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed);
|
||||
EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
|
||||
}
|
||||
|
||||
{
|
||||
auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent");
|
||||
EXPECT_FALSE(accessor->maybeLstat(CanonPath::root));
|
||||
EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError);
|
||||
}
|
||||
|
||||
{
|
||||
auto accessor = makeFSSourceAccessor(tmpDir, true);
|
||||
EXPECT_EQ(accessor->getLastModified(), 0);
|
||||
accessor->maybeLstat(CanonPath("file1"));
|
||||
EXPECT_GT(accessor->getLastModified(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
Loaded 100 of 147 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user