Compare commits
301
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0bc58d5501 | ||
|
|
94907eb37a | ||
|
|
2f1ce8900b | ||
|
|
4c6ad728d0 | ||
|
|
83360cd7b7 | ||
|
|
2a21bd6d0a | ||
|
|
5570c31b30 | ||
|
|
eead36de18 | ||
|
|
dae41e06e8 | ||
|
|
daba5f6386 | ||
|
|
6f0fe5636d | ||
|
|
73a727f3d2 | ||
|
|
cc8f4912f5 | ||
|
|
6276642164 | ||
|
|
fbd787b910 | ||
|
|
751a0f40bc | ||
|
|
857a2053ad | ||
|
|
087b6c4dc2 | ||
|
|
3b8c408108 | ||
|
|
b7ddbb8e2d | ||
|
|
fcfa1dc8ab | ||
|
|
ba219cb047 | ||
|
|
9fa69276c4 | ||
|
|
98f6881d11 | ||
|
|
56c9d5f04e | ||
|
|
07a3171fb9 | ||
|
|
52100c6ee1 | ||
|
|
fa53a9cec8 | ||
|
|
44dce7a3d1 | ||
|
|
12ef043655 | ||
|
|
7a40df3510 | ||
|
|
4f733f736e | ||
|
|
c6d07ec0aa | ||
|
|
e2cd5679eb | ||
|
|
1555677cd5 | ||
|
|
6dd89b5a2a | ||
|
|
73beff89cb | ||
|
|
f429d8d4aa | ||
|
|
017a247e63 | ||
|
|
4a267f720e | ||
|
|
490f6eeba5 | ||
|
|
c398dd7cbd | ||
|
|
37834c5e58 | ||
|
|
67a99db5be | ||
|
|
a59bc630aa | ||
|
|
6ba067831a | ||
|
|
b24df97a11 | ||
|
|
556974f33b | ||
|
|
b40b786839 | ||
|
|
9e496f9af2 | ||
|
|
726e924bd7 | ||
|
|
3a421388dd | ||
|
|
a32c139379 | ||
|
|
63344d3a3b | ||
|
|
4db68c28c1 | ||
|
|
6ba468805b | ||
|
|
d25ab60d66 | ||
|
|
656e1fc659 | ||
|
|
054de385d8 | ||
|
|
3256aba6a2 | ||
|
|
b3df7f8a3d | ||
|
|
af7c7b6723 | ||
|
|
d6fa3e3b50 | ||
|
|
5d2938520c | ||
|
|
51ca872c60 | ||
|
|
1e1d9f28ba | ||
|
|
54c62782f5 | ||
|
|
162b0072a7 | ||
|
|
eb653e5928 | ||
|
|
33ce87276f | ||
|
|
f1d468e840 | ||
|
|
3fb0fb00ae | ||
|
|
3374fdc04a | ||
|
|
a9b1a52623 | ||
|
|
f162bb62f7 | ||
|
|
292156c336 | ||
|
|
3cb42b581a | ||
|
|
17295066e8 | ||
|
|
036738be11 | ||
|
|
3c3ceb18e9 | ||
|
|
68f549def4 | ||
|
|
3fd85c7d64 | ||
|
|
25eb07a91b | ||
|
|
0a632cbc3a | ||
|
|
8c2021989e | ||
|
|
92344a31fa | ||
|
|
77fa94d8d4 | ||
|
|
ee64ffcd75 | ||
|
|
8012b584c3 | ||
|
|
609a4e999f | ||
|
|
6fd36553b9 | ||
|
|
ce28cb32e9 | ||
|
|
3d18d73003 | ||
|
|
8089af3bb0 | ||
|
|
b49ea8b246 | ||
|
|
af821ba647 | ||
|
|
1b1c949d0c | ||
|
|
fb70a45b9e | ||
|
|
53e942bfcc | ||
|
|
4f5172ba21 | ||
|
|
1bddbff3a6 | ||
|
|
4991defc44 | ||
|
|
8f829e478f | ||
|
|
7ac5a61208 | ||
|
|
2af5792cc2 | ||
|
|
e251ffdd46 | ||
|
|
a8087ebf52 | ||
|
|
66fefcd795 | ||
|
|
1ea3a841bb | ||
|
|
fd0bcd97e8 | ||
|
|
aaccb73916 | ||
|
|
b19bfc6373 | ||
|
|
3b028edbf7 | ||
|
|
23a7178eb4 | ||
|
|
252aff5c8f | ||
|
|
920c5ceb0c | ||
|
|
de76cb681d | ||
|
|
d19bfb0045 | ||
|
|
160822858a | ||
|
|
25998fcd1e | ||
|
|
7ba09399cc | ||
|
|
f8a92564f7 | ||
|
|
2ae4121c1d | ||
|
|
38c755f168 | ||
|
|
cb2ade20d4 | ||
|
|
08887caa1a | ||
|
|
8d588ad471 | ||
|
|
d51ac82dd2 | ||
|
|
6970efe2e1 | ||
|
|
3b95b7c9aa | ||
|
|
4289e2f9e6 | ||
|
|
d1dc2d53b1 | ||
|
|
ccdd1f1c65 | ||
|
|
b7fd471f84 | ||
|
|
4ab2cdacfc | ||
|
|
21534baa89 | ||
|
|
477aa250d4 | ||
|
|
2417ee4732 | ||
|
|
36a6247a0b | ||
|
|
ac24ef84fa | ||
|
|
75571ec0a0 | ||
|
|
5a65b1f131 | ||
|
|
24da83853a | ||
|
|
6f7190bdae | ||
|
|
7ce871ee86 | ||
|
|
b474e8d249 | ||
|
|
9a63752317 | ||
|
|
afc6c24d68 | ||
|
|
a3043d991f | ||
|
|
7c3b4f72b8 | ||
|
|
7a5f49323d | ||
|
|
1176d59c8a | ||
|
|
05df7d716a | ||
|
|
75da37f792 | ||
|
|
07a260ca18 | ||
|
|
8ba7ebca3b | ||
|
|
931f84b720 | ||
|
|
ec12953822 | ||
|
|
22e46fb0ef | ||
|
|
9859068689 | ||
|
|
cc5a403bc4 | ||
|
|
357a45253c | ||
|
|
1807dc78d6 | ||
|
|
cf5644ce99 | ||
|
|
47ccf201fb | ||
|
|
ec65132ab9 | ||
|
|
bde0b22dc8 | ||
|
|
a8e8614fd3 | ||
|
|
6a32b754a9 | ||
|
|
43a472631b | ||
|
|
eff403b5ab | ||
|
|
6e6dda9f67 | ||
|
|
ee1383f75f | ||
|
|
5be07abf6d | ||
|
|
469212bd38 | ||
|
|
4a1a562461 | ||
|
|
6c884fff0c | ||
|
|
bf3638376f | ||
|
|
d022cb61f2 | ||
|
|
eaf474bf24 | ||
|
|
31bffd3c78 | ||
|
|
7610d07601 | ||
|
|
ab7c0ae4a3 | ||
|
|
fef2e2e314 | ||
|
|
644be074e1 | ||
|
|
b75403f15b | ||
|
|
28c7e42ab5 | ||
|
|
401fbe3981 | ||
|
|
e44e1cc99c | ||
|
|
0900638f1d | ||
|
|
df74624754 | ||
|
|
73a7962073 | ||
|
|
f129bbb9e9 | ||
|
|
4b8991256a | ||
|
|
843395a2c8 | ||
|
|
fc52891b44 | ||
|
|
04c0e3432a | ||
|
|
dd75397f73 | ||
|
|
2c55c4aae4 | ||
|
|
80c9ad7de4 | ||
|
|
24610d51f4 | ||
|
|
4db99ea955 | ||
|
|
84ff2ef347 | ||
|
|
3933e45d52 | ||
|
|
a1569458cc | ||
|
|
4599daa10e | ||
|
|
6cb8b58a47 | ||
|
|
d19b8d5f99 | ||
|
|
0068d58b18 | ||
|
|
08bcd70ecb | ||
|
|
b5e039b7c1 | ||
|
|
e8080379ee | ||
|
|
56ab28c1ad | ||
|
|
b066db6011 | ||
|
|
8aa02c19b6 | ||
|
|
b2fdacd36c | ||
|
|
edebabe9bf | ||
|
|
b17034ba59 | ||
|
|
5aa2af1354 | ||
|
|
c54af23b41 | ||
|
|
6eebfe6274 | ||
|
|
c867ed6726 | ||
|
|
fb05f6de0d | ||
|
|
745983dfc0 | ||
|
|
8093df5255 | ||
|
|
b813ed2602 | ||
|
|
f6ca5dc5cb | ||
|
|
cfe5fc6a4a | ||
|
|
0b5773d1d0 | ||
|
|
212bf2b702 | ||
|
|
a3bcd2543e | ||
|
|
76c6f3cfd0 | ||
|
|
3c3e5cbcdb | ||
|
|
130a656330 | ||
|
|
396358dc08 | ||
|
|
d85e5dfa60 | ||
|
|
96204ea6bd | ||
|
|
c6ac52da70 | ||
|
|
b168ec2277 | ||
|
|
cc88e1aa82 | ||
|
|
f457245a9a | ||
|
|
ce16d6fdd3 | ||
|
|
b6dd17d6f2 | ||
|
|
b1a230de75 | ||
|
|
bec436c0b1 | ||
|
|
f7fc24c973 | ||
|
|
63cfefd6cb | ||
|
|
fefcc4c7cc | ||
|
|
a720cb0656 | ||
|
|
32a79fcbbf | ||
|
|
132a93625b | ||
|
|
5cf1c0ebca | ||
|
|
6b52fa8360 | ||
|
|
19a2493132 | ||
|
|
6bea8e0e08 | ||
|
|
8104858643 | ||
|
|
a6c1d5637a | ||
|
|
f1f99b6598 | ||
|
|
2308f200c8 | ||
|
|
048d0b6781 | ||
|
|
0c8751d3f4 | ||
|
|
723c47550e | ||
|
|
66c867395f | ||
|
|
27006cc8a9 | ||
|
|
06f21596a0 | ||
|
|
9254fab407 | ||
|
|
7541129f04 | ||
|
|
994324feda | ||
|
|
1f739961e5 | ||
|
|
cccfa385e6 | ||
|
|
9d2100a165 | ||
|
|
4769f3c0b2 | ||
|
|
ab354dc8f6 | ||
|
|
188cb798ad | ||
|
|
3cc07ede73 | ||
|
|
1aa7ab0dcf | ||
|
|
208ed3c538 | ||
|
|
b6add8dcc6 | ||
|
|
79750a3ccc | ||
|
|
30cd9e43e1 | ||
|
|
0695630eb5 | ||
|
|
89dc57f6aa | ||
|
|
f274a7273a | ||
|
|
675656ffba | ||
|
|
a5edc2d921 | ||
|
|
97e3816b24 | ||
|
|
7998508a40 | ||
|
|
a1e24fa6ce | ||
|
|
21a251be5f | ||
|
|
5694772794 | ||
|
|
720027470d | ||
|
|
18fece25cd | ||
|
|
cc1edfaf5b | ||
|
|
fd1ecfbfc8 | ||
|
|
c2d2a0fe2d | ||
|
|
4167686789 | ||
|
|
075242b096 | ||
|
|
3965b6889a | ||
|
|
03ca960c14 | ||
|
|
651dc72506 | ||
|
|
5166cee704 |
No files matched your search
@@ -24,8 +24,8 @@ inputs:
|
||||
description: "Github token"
|
||||
required: true
|
||||
use_cache:
|
||||
description: "Whether to setup magic-nix-cache"
|
||||
default: true
|
||||
description: "Whether to setup github actions cache (not implemented currently)"
|
||||
default: false
|
||||
required: false
|
||||
runs:
|
||||
using: "composite"
|
||||
@@ -122,10 +122,3 @@ runs:
|
||||
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
|
||||
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
|
||||
extra-conf: ${{ inputs.extra_nix_config }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
|
||||
if: ${{ inputs.use_cache == 'true' }}
|
||||
with:
|
||||
diagnostic-endpoint: ''
|
||||
use-flakehub: false
|
||||
use-gha-cache: true
|
||||
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
|
||||
+15
-81
@@ -164,7 +164,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Download installer tarball
|
||||
uses: actions/download-artifact@v7
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
with:
|
||||
name: installer-${{matrix.os}}
|
||||
path: out
|
||||
@@ -197,80 +197,6 @@ jobs:
|
||||
- run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
|
||||
- run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
|
||||
|
||||
# Steps to test CI automation in your own fork.
|
||||
# 1. Sign-up for https://hub.docker.com/
|
||||
# 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
|
||||
# 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
|
||||
check_secrets:
|
||||
permissions:
|
||||
contents: none
|
||||
name: Check presence of secrets
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
docker: ${{ steps.secret.outputs.docker }}
|
||||
steps:
|
||||
- name: Check for DockerHub secrets
|
||||
id: secret
|
||||
env:
|
||||
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
|
||||
|
||||
docker_push_image:
|
||||
needs: [tests, check_secrets]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
if: >-
|
||||
needs.check_secrets.outputs.docker == 'true' &&
|
||||
github.event_name == 'push' &&
|
||||
github.ref_name == 'master'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
dogfood: false
|
||||
extra_nix_config: |
|
||||
experimental-features = flakes nix-command
|
||||
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
|
||||
- run: nix build .#dockerImage -L
|
||||
- run: docker load -i ./result/image.tar.gz
|
||||
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
|
||||
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
|
||||
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
|
||||
#
|
||||
# Push to Docker Hub first
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
|
||||
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
|
||||
# Push to GitHub Container Registry as well
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Push image
|
||||
run: |
|
||||
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
|
||||
# Change all uppercase to lowercase
|
||||
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
|
||||
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:latest
|
||||
docker push $IMAGE_ID:$NIX_VERSION
|
||||
docker push $IMAGE_ID:latest
|
||||
# deprecated 2024-02-24
|
||||
docker tag nix:$NIX_VERSION $IMAGE_ID:master
|
||||
docker push $IMAGE_ID:master
|
||||
|
||||
flake_regressions:
|
||||
needs: tests
|
||||
runs-on: ubuntu-24.04
|
||||
@@ -287,13 +213,21 @@ jobs:
|
||||
with:
|
||||
repository: NixOS/flake-regressions-data
|
||||
path: flake-regressions/tests
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
- name: Download installer tarball
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
with:
|
||||
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
|
||||
extra_nix_config:
|
||||
experimental-features = nix-command flakes
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
|
||||
name: installer-linux
|
||||
path: out
|
||||
- name: Looking up the installer tarball URL
|
||||
id: installer-tarball-url
|
||||
run: |
|
||||
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
||||
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
|
||||
with:
|
||||
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
- name: Run flake regressions tests
|
||||
run: MAX_FLAKES=25 flake-regressions/eval-all.sh
|
||||
|
||||
profile_build:
|
||||
needs: tests
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
name: Upload Release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
eval_id:
|
||||
description: "Hydra evaluation ID"
|
||||
required: true
|
||||
type: number
|
||||
is_latest:
|
||||
description: "Mark as latest release"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-24.04
|
||||
environment: releases
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- uses: ./.github/actions/install-nix-action
|
||||
with:
|
||||
dogfood: false # Use stable version
|
||||
use_cache: false # Don't want any cache injection shenanigans
|
||||
extra_nix_config: |
|
||||
experimental-features = nix-command flakes
|
||||
- name: Set NIX_PATH from flake input
|
||||
run: |
|
||||
NIXPKGS_PATH=$(nix build --inputs-from .# nixpkgs#path --print-out-paths --no-link)
|
||||
# Shebangs with perl have issues. Pin nixpkgs this way. nix shell should maybe
|
||||
# get the same uberhack that nix-shell has to support it.
|
||||
echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV"
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
|
||||
with:
|
||||
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
|
||||
role-session-name: nix-release-oidc-${{ github.run_id }}
|
||||
aws-region: eu-west-1
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Upload release
|
||||
run: |
|
||||
./maintainers/upload-release.pl \
|
||||
${{ inputs.eval_id }} \
|
||||
--skip-git
|
||||
env:
|
||||
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
|
||||
- name: Push to GHCR
|
||||
run: |
|
||||
DOCKER_OWNER="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[A-Z]' '[a-z]')/nix"
|
||||
./maintainers/upload-release.pl \
|
||||
${{ inputs.eval_id }} \
|
||||
--skip-git \
|
||||
--skip-s3 \
|
||||
--docker-owner "$DOCKER_OWNER"
|
||||
env:
|
||||
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
|
||||
@@ -26,7 +26,6 @@ bash = find_program('bash', native : true)
|
||||
# HTML manual dependencies (conditional)
|
||||
if get_option('html-manual')
|
||||
mdbook = find_program('mdbook', native : true)
|
||||
rsync = find_program('rsync', required : true, native : true)
|
||||
endif
|
||||
|
||||
pymod = import('python')
|
||||
@@ -126,7 +125,12 @@ if get_option('html-manual')
|
||||
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
|
||||
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
|
||||
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
|
||||
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
|
||||
# Copy source to build directory, excluding the build directory itself
|
||||
# (which is present when built as an individual component).
|
||||
# Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests).
|
||||
(cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -)
|
||||
chmod -R u+w @2@
|
||||
find @2@ -name '*.drv' -delete
|
||||
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
|
||||
rm -rf @2@/manual
|
||||
mv @2@/html @2@/manual
|
||||
@@ -138,7 +142,6 @@ if get_option('html-manual')
|
||||
mdbook.full_path(),
|
||||
meson.current_build_dir(),
|
||||
meson.project_version(),
|
||||
rsync.full_path(),
|
||||
),
|
||||
],
|
||||
input : [
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
mdbook,
|
||||
jq,
|
||||
python3,
|
||||
rsync,
|
||||
nix-cli,
|
||||
changelog-d,
|
||||
json-schema-for-humans,
|
||||
@@ -54,6 +53,8 @@ mkMesonDerivation (finalAttrs: {
|
||||
../../src/libstore-tests/data/nar-info
|
||||
../../src/libstore-tests/data/build-result
|
||||
../../src/libstore-tests/data/dummy-store
|
||||
# For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/
|
||||
../../tests/functional/derivation
|
||||
# Too many different types of files to filter for now
|
||||
../../doc/manual
|
||||
./.
|
||||
@@ -90,7 +91,6 @@ mkMesonDerivation (finalAttrs: {
|
||||
]
|
||||
++ lib.optionals buildHtmlManual [
|
||||
mdbook
|
||||
rsync
|
||||
json-schema-for-humans
|
||||
]
|
||||
++ lib.optionals (!officialRelease && buildHtmlManual) [
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
synopsis: "C API: New store API methods"
|
||||
prs: [14766]
|
||||
---
|
||||
|
||||
The C API now includes additional methods:
|
||||
|
||||
- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
synopsis: "New setting `ignore-gc-delete-failure` for local stores"
|
||||
prs: [15054]
|
||||
---
|
||||
|
||||
A new local store setting `ignore-gc-delete-failure` has been added.
|
||||
When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths.
|
||||
This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store.
|
||||
|
||||
This setting is experimental and requires `--extra-experimental-features local-overlay-store`.
|
||||
@@ -135,7 +135,9 @@
|
||||
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
|
||||
- [Store Path Specification](protocols/store-path.md)
|
||||
- [Nix Archive (NAR) Format](protocols/nix-archive/index.md)
|
||||
- [Nix Cache Info Format](protocols/nix-cache-info.md)
|
||||
- [Derivation "ATerm" file format](protocols/derivation-aterm.md)
|
||||
- [Nix32 Encoding](protocols/nix32.md)
|
||||
- [C API](c-api.md)
|
||||
- [Glossary](glossary.md)
|
||||
- [Development](development/index.md)
|
||||
|
||||
@@ -27,7 +27,7 @@ site](https://en.wikipedia.org/wiki/Call_site) position and the name of the
|
||||
function being called (when available). For example:
|
||||
|
||||
```
|
||||
/nix/store/x9wnkly3k1gkq580m90jjn32q9f05q2v-source/pkgs/top-level/default.nix:167:5:primop import
|
||||
/nix/store/2q71fdvr4h33g9832hiriwnf20fn630l-source/pkgs/top-level/default.nix:167:5:primop import
|
||||
```
|
||||
|
||||
Here `import` primop is called at `/nix/store/x9wnkly3k1gkq580m90jjn32q9f05q2v-source/pkgs/top-level/default.nix:167:5`.
|
||||
Here `import` primop is called at `/nix/store/2q71fdvr4h33g9832hiriwnf20fn630l-source/pkgs/top-level/default.nix:167:5`.
|
||||
@@ -57,11 +57,6 @@ Most Nix commands interpret the following environment variables:
|
||||
|
||||
Overrides the location of the Nix store (default `prefix/store`).
|
||||
|
||||
- <span id="env-NIX_DATA_DIR">[`NIX_DATA_DIR`](#env-NIX_DATA_DIR)</span>
|
||||
|
||||
Overrides the location of the Nix static data directory (default
|
||||
`prefix/share`).
|
||||
|
||||
- <span id="env-NIX_LOG_DIR">[`NIX_LOG_DIR`](#env-NIX_LOG_DIR)</span>
|
||||
|
||||
Overrides the location of the Nix log directory (default
|
||||
|
||||
@@ -39,11 +39,11 @@ This makes all subscribed channels available as attributes in the default expres
|
||||
A symlink that ensures that [`nix-env`] can find the current user's [channels]:
|
||||
|
||||
- `~/.nix-defexpr/channels`
|
||||
- `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`.
|
||||
- `$XDG_STATE_HOME/nix/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`.
|
||||
|
||||
This symlink points to:
|
||||
|
||||
- `$XDG_STATE_HOME/profiles/channels` for regular users
|
||||
- `$XDG_STATE_HOME/nix/profiles/channels` for regular users
|
||||
- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root`
|
||||
|
||||
In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.
|
||||
|
||||
@@ -114,9 +114,9 @@ Here is an example of how this file might look like after installing `hello` fro
|
||||
};
|
||||
name = "hello-2.12.1";
|
||||
out = {
|
||||
outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1";
|
||||
outPath = "/nix/store/src1vzij2z0slnakrsbpqpk20389z0k6-hello-2.12.1";
|
||||
};
|
||||
outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1";
|
||||
outPath = "/nix/store/src1vzij2z0slnakrsbpqpk20389z0k6-hello-2.12.1";
|
||||
outputs = [ "out" ];
|
||||
system = "x86_64-linux";
|
||||
type = "derivation";
|
||||
|
||||
@@ -37,13 +37,13 @@ dr-xr-xr-x 4 root root 4096 Jan 1 1970 share
|
||||
|
||||
/home/eelco/.local/state/nix/profiles/profile-7-link/bin:
|
||||
total 20
|
||||
lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
|
||||
lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/cyxny9d1zjb9l9103fr6j6kavp3bqjxf-chromium-86.0.4240.111/bin/chromium
|
||||
lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
|
||||
lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
|
||||
|
||||
/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications:
|
||||
total 12
|
||||
lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
|
||||
lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/sqzyx2l85i6j2a77pnyvglh3bvzwmjjp-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
|
||||
lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
|
||||
lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
|
||||
|
||||
|
||||
@@ -72,11 +72,11 @@ When using public key authentication, you can avoid typing the passphrase with `
|
||||
> $ storePath="$(nix-build '<nixpkgs>' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)"
|
||||
> $ nix-copy-closure --to alice@itchy.example.org "$storePath"
|
||||
> copying 5 paths...
|
||||
> copying path '/nix/store/nrwkk6ak3rgkrxbqhsscb01jpzmslf2r-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/gm61h1y42pqyl6178g90x8zm22n6pyy5-libunistring-1.1' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/ddfzjdykw67s20c35i7a6624by3iz5jv-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/apab5i73dqa09wx0q27b6fbhd1r18ihl-glibc-2.39-31' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/g1n2vryg06amvcc1avb2mcq36faly0mh-hello-2.12.1' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/h6q8sqsqfbd3252f9gixqn3z282wds7m-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/imnwvn96lw355giswsk36hx105j4wnpj-libunistring-1.1' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/85301indj7scg34spnfczkz72jgv8wa9-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/ypwfsaljwhzw9iffiysxmxnhjj8v7np0-glibc-2.39-31' to 'ssh://alice@itchy.example.org'...
|
||||
> copying path '/nix/store/0dklv59zppdsqdvgf0qdvjgzcs5wbwxa-hello-2.12.1' to 'ssh://alice@itchy.example.org'...
|
||||
> ```
|
||||
|
||||
> **Example**
|
||||
|
||||
@@ -204,7 +204,7 @@ To install a specific [store derivation] (typically created by
|
||||
`nix-instantiate`):
|
||||
|
||||
```console
|
||||
$ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
|
||||
$ nix-env --install /nix/store/8la6y31fmm6i4wfmby6avly1wf718xnj-gcc-3.4.3.drv
|
||||
```
|
||||
|
||||
To install a specific output path:
|
||||
@@ -232,7 +232,7 @@ $ nix-env --file '<nixpkgs>' --install --attr hello --dry-run
|
||||
(dry run; not doing anything)
|
||||
installing ‘hello-2.10’
|
||||
this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
|
||||
/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
|
||||
/nix/store/ikwkxz4wwlp2g1428n7dy729cg1d9hin-hello-2.10
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ $ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
|
||||
```console
|
||||
$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
|
||||
0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
|
||||
/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
|
||||
/nix/store/8alrpdaasjd1x6g1fczchmzbpqm936a3-hello-2.10.tar.gz
|
||||
```
|
||||
|
||||
```console
|
||||
|
||||
@@ -34,6 +34,6 @@ This operation has the following options:
|
||||
|
||||
```console
|
||||
$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
|
||||
/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
|
||||
/nix/store/8alrpdaasjd1x6g1fczchmzbpqm936a3-hello-2.10.tar.gz
|
||||
```
|
||||
|
||||
@@ -27,7 +27,7 @@ paths in the store that refer to it (i.e., depend on it).
|
||||
# Example
|
||||
|
||||
```console
|
||||
$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4
|
||||
$ nix-store --delete /nix/store/gjak3al7lj61x4gj6rln4f5pc5v0f67n-mesa-6.4
|
||||
0 bytes freed (0.00 MiB)
|
||||
error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive
|
||||
error: cannot delete path `/nix/store/gjak3al7lj61x4gj6rln4f5pc5v0f67n-mesa-6.4' since it is still alive
|
||||
```
|
||||
@@ -184,9 +184,9 @@ Print the build-time dependencies of `svn`:
|
||||
|
||||
```console
|
||||
$ nix-store --query --requisites $(nix-store --query --deriver $(which svn))
|
||||
/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
|
||||
/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
|
||||
/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
|
||||
/nix/store/y6qa66l9h0pw161crnlk6y16rdrcljx4-grep-2.5.1.tar.bz2.drv
|
||||
/nix/store/z716h753s97jhnzvfank2srqbljswpgm-gcc-wrapper.sh
|
||||
/nix/store/f39x0q73rjdyvzm93y9wrkfr6x39lb7f-glibc-2.3.4.drv
|
||||
... lots of other paths ...
|
||||
```
|
||||
|
||||
@@ -199,10 +199,10 @@ Show the build-time dependencies as a tree:
|
||||
```console
|
||||
$ nix-store --query --tree $(nix-store --query --deriver $(which svn))
|
||||
/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
|
||||
+---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
|
||||
+---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
|
||||
| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
|
||||
| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
|
||||
+---/nix/store/vxnmkc8l8d2ijjha4xwhkfgx9vvc3q4c-builder.sh
|
||||
+---/nix/store/rn9776dy82n5qrgz7xbcl1iw4vfkcrkk-bash-3.0.drv
|
||||
| +---/nix/store/x9j20hz6bln1crzn55qifk0bbsm8v5ac-bash
|
||||
| +---/nix/store/ajnn1mcm45wjvn0rlc22gvx2cwhjnazx-builder.sh
|
||||
...
|
||||
```
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ This operation is typically used to build [store derivation]s produced by
|
||||
|
||||
```console
|
||||
$ nix-store --realise $(nix-instantiate ./test.nix)
|
||||
/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
|
||||
/nix/store/6gwmy5jcnwdlz6aqqhksz863f1l8xc2w-aterm-2.3.1
|
||||
```
|
||||
|
||||
This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does.
|
||||
|
||||
@@ -136,7 +136,7 @@
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
|
||||
> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1`
|
||||
|
||||
See [Store Path](@docroot@/store/store-path.md) for details.
|
||||
|
||||
|
||||
@@ -6,14 +6,23 @@ It is broken up into multiple Meson packages, which are optionally combined in a
|
||||
There are no mandatory extra steps to the building process:
|
||||
generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work.
|
||||
|
||||
The installation path can be specified by passing the `-Dprefix=prefix`
|
||||
to `configure`. The default installation directory is `/usr/local`. You
|
||||
```bash
|
||||
git clone https://github.com/NixOS/nix.git
|
||||
cd nix
|
||||
meson setup build
|
||||
cd build
|
||||
ninja
|
||||
(sudo) ninja install
|
||||
```
|
||||
|
||||
The installation path can be specified by passing `-Dprefix=prefix`
|
||||
to `meson setup build`. The default installation directory is `/usr/local`. You
|
||||
can change this to any location you like. You must have write permission
|
||||
to the *prefix* path.
|
||||
|
||||
Nix keeps its *store* (the place where packages are stored) in
|
||||
`/nix/store` by default. This can be changed using
|
||||
`-Dstore-dir=path`.
|
||||
`-Dlibstore:store-dir=path`.
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
|
||||
@@ -8,7 +8,7 @@ stores packages in the _Nix store_, usually the directory
|
||||
`/nix/store`, where each package has its own unique subdirectory such
|
||||
as
|
||||
|
||||
/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/
|
||||
/nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/
|
||||
|
||||
where `b6gvzjyb2pg0…` is a unique identifier for the package that
|
||||
captures all its dependencies (it’s a cryptographic hash of the
|
||||
|
||||
@@ -338,7 +338,7 @@ Here is more information on the `output*` attributes, and what values they may b
|
||||
This will specify the output hash of the single output of a [fixed-output derivation].
|
||||
|
||||
The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/).
|
||||
The "nix32" encoding is an adaptation of base-32 encoding.
|
||||
The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
|
||||
@@ -34,12 +34,12 @@ String context elements come in different forms:
|
||||
> [`builtins.storePath`] creates a string with a single constant string context element:
|
||||
>
|
||||
> ```nix
|
||||
> builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10")
|
||||
> builtins.getContext (builtins.storePath "/nix/store/ikwkxz4wwlp2g1428n7dy729cg1d9hin-hello-2.10")
|
||||
> ```
|
||||
> evaluates to
|
||||
> ```nix
|
||||
> {
|
||||
> "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = {
|
||||
> "/nix/store/ikwkxz4wwlp2g1428n7dy729cg1d9hin-hello-2.10" = {
|
||||
> path = true;
|
||||
> };
|
||||
> }
|
||||
|
||||
@@ -181,7 +181,7 @@ A derivation interpolates to the [store path] of its first [output](./derivation
|
||||
> "${pkgs.hello}"
|
||||
> ```
|
||||
>
|
||||
> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1"
|
||||
> "/nix/store/qnlr7906z0mrl2syrkdbpicffq02nw07-hello-2.12.1"
|
||||
|
||||
An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself.
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ See [String literals](string-literals.md).
|
||||
|
||||
Path literals can also include [string interpolation], besides being [interpolated into other expressions].
|
||||
|
||||
[string interpolation]: ./string-interpolation.md
|
||||
[interpolated into other expressions]: ./string-interpolation.md#interpolated-expression
|
||||
|
||||
At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.
|
||||
@@ -272,7 +273,7 @@ will crash with an `infinite recursion encountered` error message.
|
||||
|
||||
A let-expression allows you to define local variables for an expression.
|
||||
|
||||
> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr*
|
||||
> *let-in* = `let` [ *identifier* = *expr* `;` ]... `in` *expr*
|
||||
|
||||
Example:
|
||||
|
||||
@@ -285,6 +286,27 @@ in x + y
|
||||
|
||||
This evaluates to `"foobar"`.
|
||||
|
||||
There is also another, older, syntax for let expressions that should not be used in new code:
|
||||
|
||||
> *let* = `let` `{` *identifier* = *expr* `;` [ *identifier* = *expr* `;`]... `}`
|
||||
|
||||
In this form, the attribute set between the `{` `}` is recursive.
|
||||
|
||||
One of the attributes must have the special name `body`,
|
||||
which is the result of the expression.
|
||||
|
||||
Example:
|
||||
|
||||
```nix
|
||||
let {
|
||||
foo = bar;
|
||||
bar = "baz";
|
||||
body = foo;
|
||||
}
|
||||
```
|
||||
|
||||
This evaluates to "baz".
|
||||
|
||||
## Inheriting attributes
|
||||
|
||||
When defining an [attribute set](./types.md#type-attrs) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes).
|
||||
|
||||
@@ -19,17 +19,16 @@ whatever port you like:
|
||||
$ nix-serve -p 8080
|
||||
```
|
||||
|
||||
To check whether it works, try the following on the client:
|
||||
To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client:
|
||||
|
||||
```console
|
||||
$ curl http://avalon:8080/nix-cache-info
|
||||
StoreDir: /nix/store
|
||||
WantMassQuery: 1
|
||||
Priority: 30
|
||||
```
|
||||
|
||||
which should print something like:
|
||||
|
||||
StoreDir: /nix/store
|
||||
WantMassQuery: 1
|
||||
Priority: 30
|
||||
When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist.
|
||||
|
||||
On the client side, you can tell Nix to use your binary cache using
|
||||
`--substituters`, e.g.:
|
||||
|
||||
@@ -94,8 +94,8 @@ properties:
|
||||
>
|
||||
> ```json
|
||||
> "srcs": [
|
||||
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
|
||||
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
|
||||
> "b8nwz167km1yciqpwzjj24f8jcy8pq1h-separate-debug-info.sh",
|
||||
> "ihzmilr413r8fb3ah30yjnhlb18c1laz-fix-pop-var-context-error.patch"
|
||||
> ]
|
||||
> ```
|
||||
items:
|
||||
@@ -140,7 +140,7 @@ properties:
|
||||
description: |
|
||||
Absolute path of the program used to perform the build.
|
||||
Typically this is the `bash` shell
|
||||
(e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
|
||||
(e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`).
|
||||
|
||||
args:
|
||||
type: array
|
||||
|
||||
@@ -24,7 +24,7 @@ description: |
|
||||
|
||||
The format follows this pattern: `${digest}-${name}`
|
||||
|
||||
- **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
|
||||
- **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
|
||||
- **name**: The package name and optional version/suffix information
|
||||
|
||||
type: string
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Nix Cache Info Format
|
||||
|
||||
The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`).
|
||||
|
||||
MIME type: `text/x-nix-cache-info`
|
||||
|
||||
## Format
|
||||
|
||||
Line-based key-value format:
|
||||
|
||||
```
|
||||
Key: value
|
||||
```
|
||||
|
||||
Leading and trailing whitespace is trimmed from values.
|
||||
Lines without a colon are ignored.
|
||||
Unknown keys are silently ignored.
|
||||
|
||||
## Fields
|
||||
|
||||
### `StoreDir`
|
||||
|
||||
The Nix store directory path that this cache was built for (e.g., `/nix/store`).
|
||||
|
||||
If present, Nix verifies that this matches the client's store directory:
|
||||
|
||||
```
|
||||
error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/store', not '/home/user/nix/store'
|
||||
```
|
||||
|
||||
### `WantMassQuery`
|
||||
|
||||
`1` or `0`. Sets the default for [`want-mass-query`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-want-mass-query).
|
||||
|
||||
### `Priority`
|
||||
|
||||
Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority).
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
StoreDir: /nix/store
|
||||
WantMassQuery: 1
|
||||
Priority: 30
|
||||
```
|
||||
|
||||
## Caching Behavior
|
||||
|
||||
Nix caches `nix-cache-info` in the [cache directory](@docroot@/command-ref/env-common.md#env-NIX_CACHE_HOME) with a 7-day TTL.
|
||||
|
||||
## See Also
|
||||
|
||||
- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md)
|
||||
- [Serving a Nix store via HTTP](@docroot@/package-management/binary-cache-substituter.md)
|
||||
- [`substituters`](@docroot@/command-ref/conf-file.md#conf-substituters)
|
||||
@@ -0,0 +1,19 @@
|
||||
# Nix32 Encoding
|
||||
|
||||
Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute.
|
||||
|
||||
## Alphabet
|
||||
|
||||
The Nix32 alphabet consists of these 32 characters:
|
||||
|
||||
```
|
||||
0 1 2 3 4 5 6 7 8 9 a b c d f g h i j k l m n p q r s v w x y z
|
||||
```
|
||||
|
||||
The letters `e`, `o`, `u`, and `t` are omitted.
|
||||
|
||||
## Byte Order
|
||||
|
||||
Nix32 encoding processes the hash bytes from the end (last byte first), while base-16 encoding processes from the beginning (first byte first).
|
||||
|
||||
Consequently, the string sort order is determined primarily by the first bytes for base-16, and by the last bytes for Nix32.
|
||||
@@ -20,12 +20,11 @@ where
|
||||
|
||||
- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory)
|
||||
|
||||
- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`
|
||||
- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`.
|
||||
|
||||
For the definition of the hash compression algorithm, please refer to the section 5.1 of
|
||||
the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf), which also defines the
|
||||
specifics of base-32 encoding. Note that base-32 encoding processes the hash bytestring from
|
||||
the end, while base-16 processes in from the beginning.
|
||||
Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md).
|
||||
|
||||
For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf).
|
||||
|
||||
## Fingerprint
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@
|
||||
...
|
||||
the following paths will be downloaded/copied (30.02 MiB):
|
||||
/nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
|
||||
/nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
|
||||
/nix/store/spc1m987vlibchdx369qwa391s738s7l-libunwind-0.98.6
|
||||
...
|
||||
|
||||
- Language features:
|
||||
|
||||
@@ -63,7 +63,7 @@ Nix 0.8 has the following improvements:
|
||||
can query all paths that directly or indirectly use a certain Glibc:
|
||||
|
||||
$ nix-store -q --referrers-closure \
|
||||
/nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
|
||||
/nix/store/1a6mdrjz4wn7b9sfmcw5ggbk1mi281mh-glibc-2.3.4
|
||||
|
||||
- The concept of fixed-output derivations has been formalised.
|
||||
Previously, functions such as `fetchurl` in Nixpkgs used a hack
|
||||
|
||||
@@ -66,7 +66,7 @@ This release has the following new features:
|
||||
|
||||
nix copy --to ssh://machine nixpkgs.hello
|
||||
|
||||
nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
|
||||
nix copy --to ssh://machine /nix/store/qbhyj3blxpw2i6pb7c6grc9185nbnpvy-hello-2.10
|
||||
|
||||
nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)'
|
||||
|
||||
@@ -187,7 +187,7 @@ This release has the following new features:
|
||||
former is primarily useful in conjunction with remote stores,
|
||||
e.g.
|
||||
|
||||
nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
|
||||
nix ls-store --store https://cache.nixos.org/ -lR /nix/store/qbhyj3blxpw2i6pb7c6grc9185nbnpvy-hello-2.10
|
||||
|
||||
lists the contents of path in a binary cache.
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
* Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables.
|
||||
For example,
|
||||
```shell-session
|
||||
# nix build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev
|
||||
# nix build /nix/store/fpq78s2h8ffh66v2iy0q1838mhff06y8-glibc-2.33-78.drv^dev
|
||||
```
|
||||
now works just as
|
||||
```shell-session
|
||||
|
||||
@@ -18,13 +18,13 @@
|
||||
|
||||
For example,
|
||||
```shell-session
|
||||
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
|
||||
$ nix path-info /nix/store/fpq78s2h8ffh66v2iy0q1838mhff06y8-glibc-2.33-78.drv
|
||||
```
|
||||
|
||||
now gives info about the derivation itself, while
|
||||
|
||||
```shell-session
|
||||
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
|
||||
$ nix path-info /nix/store/fpq78s2h8ffh66v2iy0q1838mhff06y8-glibc-2.33-78.drv^*
|
||||
```
|
||||
provides information about each of its outputs.
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
```json5
|
||||
[
|
||||
{
|
||||
"path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15",
|
||||
"path": "/nix/store/fvqsvk65d38p8qqir371ii0hyqxvjcw6-bash-5.2-p15",
|
||||
"valid": true,
|
||||
// ...
|
||||
},
|
||||
@@ -60,7 +60,7 @@
|
||||
|
||||
```json5
|
||||
{
|
||||
"/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": {
|
||||
"/nix/store/fvqsvk65d38p8qqir371ii0hyqxvjcw6-bash-5.2-p15": {
|
||||
// ...
|
||||
},
|
||||
"/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null,
|
||||
|
||||
@@ -182,7 +182,7 @@
|
||||
«partially applied primop map»
|
||||
|
||||
nix-repl> builtins.trace lib.id "my-value"
|
||||
trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5»
|
||||
trace: «lambda id @ /nix/store/kgr5lnaiiv08wb7k324yv1i1npjmrvjc-source/lib/trivial.nix:26:5»
|
||||
"my-value"
|
||||
```
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
|
||||
> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1`
|
||||
>
|
||||
> A rendered store path
|
||||
|
||||
@@ -22,7 +22,7 @@ Store paths are pairs of
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
|
||||
> - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq`
|
||||
> - Name: `firefox-33.1`
|
||||
|
||||
To make store objects accessible to operating system processes, stores have to expose store objects through the file system.
|
||||
@@ -31,14 +31,14 @@ A store path is rendered to a file system path as the concatenation of
|
||||
|
||||
- [Store directory](#store-directory) (typically `/nix/store`)
|
||||
- Path separator (`/`)
|
||||
- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
|
||||
- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
|
||||
- Hyphen (`-`)
|
||||
- Name
|
||||
|
||||
> **Example**
|
||||
>
|
||||
> ```
|
||||
> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
|
||||
> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1
|
||||
> |--------| |------------------------------| |----------|
|
||||
> store directory digest name
|
||||
> ```
|
||||
|
||||
@@ -8,7 +8,7 @@ Stores are specified using a URL-like syntax. For example, the command
|
||||
|
||||
```console
|
||||
# nix path-info --store https://cache.nixos.org/ --json \
|
||||
/nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1
|
||||
/nix/store/1542dip9i7k4f24y6hqgd04hmvid9hr5-coreutils-9.1
|
||||
```
|
||||
|
||||
fetches information about a store path in the HTTP binary cache
|
||||
|
||||
Generated
+5
-5
@@ -3,15 +3,15 @@
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1733328505,
|
||||
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
|
||||
"owner": "edolstra",
|
||||
"lastModified": 1767039857,
|
||||
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
|
||||
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
|
||||
inputs.flake-compat = {
|
||||
url = "github:edolstra/flake-compat";
|
||||
url = "github:NixOS/flake-compat";
|
||||
flake = false;
|
||||
};
|
||||
|
||||
@@ -418,10 +418,6 @@
|
||||
supportsCross = false;
|
||||
};
|
||||
|
||||
"nix-kaitai-struct-checks" = {
|
||||
supportsCross = false;
|
||||
};
|
||||
|
||||
"nix-perl-bindings" = {
|
||||
supportsCross = false;
|
||||
};
|
||||
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
set -x
|
||||
|
||||
git ls-files -z \
|
||||
| xargs -0 grep -o '[0123456789abcdfghijklmnpqrsvwxyz]\{32\}' 2> /dev/null \
|
||||
| rev \
|
||||
| cut -d: -f1 \
|
||||
| rev \
|
||||
| sort \
|
||||
| uniq \
|
||||
| while read -r oldhash; do
|
||||
if ! curl --fail -I "https://cache.nixos.org/$oldhash.narinfo" > /dev/null 2>&1; then
|
||||
continue
|
||||
fi
|
||||
|
||||
newhash=$(
|
||||
nix eval --expr "builtins.toFile \"006c6ssvddri1sg34wnw65mzd05pcp3qliylxlhv49binldajba5\" \"$oldhash\"" \
|
||||
| cut -d- -f1 \
|
||||
| cut -d/ -f4
|
||||
)
|
||||
|
||||
msg=$(printf "bad: %s -> %s" "$oldhash" "$newhash")
|
||||
echo "$msg"
|
||||
git ls-files -z \
|
||||
| xargs -0 grep -a -l "$oldhash" 2> /dev/null \
|
||||
| while read -r file; do
|
||||
[ -L "$file" ] && continue
|
||||
perl -pi -e "s/$oldhash/$newhash/g" "$file" || true
|
||||
done || true
|
||||
git commit -am "$msg"
|
||||
done
|
||||
@@ -0,0 +1,110 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGPtMiwBEAC0sFZW2QW/OaDjKm5zGRpDvHXDsMIUtlHfoi5ce8pocC63W05o
|
||||
FSXbUZjZ1VfYO8lT8DFANCzTkiXYaZx0cPRG2pVY4AOQZDNFt5XrAyvw496XCAIM
|
||||
DTYGFLjCqgjPt9RUFEy4MyHPJTEpB0x3rXgT4ILNu9vsj9Q0vttps7SpbZ3Ldq5H
|
||||
o/BBbLW77q/vNjpYzCbBIXF7ycUGpnNv9Go/WuiDnrBMcyxh+8kjjIHB5cxZSnjJ
|
||||
DUv681+m83v+gLZQGX/jexQrrf5JpS0X9qEnhGLrNUDhtyv5ud3Je4EfamkjLVVC
|
||||
RlNLofgflOCsl/tP80i+K7S1QdKhUALxuJ6H0prYUflGBDxDyC8XYuJ62TT0OUpa
|
||||
vJvgwVlCq8/jq+ykYQXlbuBVOzi5wAuI4l3+HqreSQYPSiwe+6N590Zbafdv1fvN
|
||||
WFtZKCTGMqfyaaAnppioH9/+NWkI2AQxaYVasYM/JEYvY9pJgA7alh51jHW4JglP
|
||||
ErypKfBKPKJID0QENqYoa3bDDCihuNWhgQf9dxzPlj2ckd35Zb6w4DfuSmtjaa9D
|
||||
o0jZVY1JbFuxBqP09+saVPrxLHgmPxjcdzPGQQtAqdO2vyJXNEGLFMoVEZPNaLo3
|
||||
QmcIJnT7oSck+4vGfOYtWUHXQynu/Tnwsv2XkA/uyw8HNe+RRMqv/apnzQARAQAB
|
||||
tCdTZXJnZWkgWmltbWVybWFuIDxzZXJnZWlAemltbWVybWFuLmZvbz6JAlEEEwEK
|
||||
ADsCGwEFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQQVim9TDqIC5fZRYRMU+upj
|
||||
RI4d+QUCaUbsaAIZAQAKCRAU+upjRI4d+VdDD/492HRaJ/8V7R7VUzkafmb2Hb28
|
||||
SLf7oiB8Uq9I7SukiDEaIT1fUhquYWQ9KWpPRNR1TX6ApXnIeuJRMGFoDVIRnmnr
|
||||
cKnYYXfqqc81VxIyKvaumB7KWbS7G4Nbor8AH1ouOOOMMS50OTJOWQA4A26inIuG
|
||||
n+7L8MeS5aT+3uNKDoTKsidC47vnaxNMcke1taPfbfo7vn69PsRCM/g9/7TQYU8b
|
||||
6xp+pM9Ao9nJneRk2YCpsGYRrWTpaik0DFKnfpPKJM/yunhtGLF2IYAp3l1mvHPK
|
||||
nnzo92zjpQuZwazEIK+23V1vRT4IjM2BewbJPAzf2/UuxEjjgNQm0tOtH2JhFNeB
|
||||
VM0BVrGxWrrwrsmv6lWghTtBc6zRWyHrj/rpjtVQNmeKYrHWJeXwVz1rqgGPmB2N
|
||||
k0MZD1UjHHhEs1Cntn7yLmxTPztRJCtR+euRu81Uo2NAvrMJ4xsDjaM0LeLTnzjV
|
||||
9AsPjD188dOFyz7VExZum4+XaaEJ41FIPLEqU3U0GAa6stEy0ylSlIN4x9aiXXVW
|
||||
xfzHHchS5jK6QAjuZxN8t01GRactNylINRf7uoTECFZTtXNqfeuk0HQxBH0LuKVE
|
||||
0PxJbcNI4mVWw1KgTJ8PUVC1IXP3sEpqPdJOYiRXgnpcS26fWOBu0aQ4mhxiaJhr
|
||||
/zBfrkLEqp20TdNDZLkCDQRj7TM1ARAAt73xO24curnHTTgXkkVMMRzcMLx3Mb1a
|
||||
2FuddxC5hzTpEpw01L91UBrXVJEg9K2KAwP5CtCLgPCqXr47Tm7krvHxWwBksgY/
|
||||
6aHRsoPQfCFUZHc0aiO+C4NCzR+aEeGKn66Oc1Hq9oUTpDgiBWhsuEPiyA1OSGF0
|
||||
4L0jeTCqfm68kWp4PIK9yuugkdDsoyj6TonuMsb3V5ctHLqop9KH+eHSkUTPo+Lk
|
||||
+bxaeAOJ1UfbohgbRbrYKAfsaghhOMDH3R1w2pvtUJz+sDbuQsiPFTqbxsXDTFws
|
||||
H4N/AQCYnnvOhqEek2sOEZ19bJXt5UrAr10mX4PGmAkWqE1JWBxpOKG3BXSGOTu1
|
||||
3dFhQfPMK+PmvUrs0kcWQr53K/aRUdKKhIfTcMfkqYTGPK5HclHph24WjXj3QFFA
|
||||
SjksQTdm6486ZmLZK4CTbAFOPfTF/aWg8gu9v4ihdq6lqHNNXxv2xBAChcd59H7p
|
||||
D5zy9z8SpwWR9V5JDmlF6HWIIau1c6lSsQq1xHvYM8EuPe03vJvor+2u/cn5zYF1
|
||||
5ZxAuPI2i5vtavg1s8ZGAAogJ9dVcP36LdJfL9quXWvmovkd//qHIepBB+l/zQio
|
||||
ZRDZlIcfV3Xycaqsb5OqHGARHE0097koipMt5y/iXlqG4Ruue6Idb8bW96EKpaWj
|
||||
kKy/iNfQfQMAEQEAAYkEcgQYAQoAJgIbAhYhBBWKb1MOogLl9lFhExT66mNEjh35
|
||||
BQJpRuz3BQkJHCDCAkDBdCAEGQEKAB0WIQRK3hK0WyJ4BicGpcmpsLVXymMjJQUC
|
||||
Y+0zNQAKCRCpsLVXymMjJbdSD/9+f1FOOeGDAJI6Duo5fsWnf4xJJdtQtDbz6d2A
|
||||
SeDapxeJ3zWfKBD0wu5sISEa0uiWsYSmLtsa2SqVAKHlEaMGRR+tkBMPQ+rvgI4c
|
||||
62YjGTgm+IPd+NFIn+ixFU1hpinTh+KhUEoeOwWCvKs9nZfSG9vkienfiG0bBxo2
|
||||
zrvBzXA50x5hbUL+ghKu/AVfN9qZDwh30O4KZTwk4g4cM9SeaQa4YvHYIS3IEhDZ
|
||||
hGybwrrqV9cs92ln4IJw9WCy9QReBNrdeFgC4+3ziUp1QsG3RvqrtuMttwBVC1Z3
|
||||
bj5QjLLOREhhodfvk98t9yVkragObb4rGrLo1mWuF0c4mJGvXwnrqhCMvzv4M+0T
|
||||
Zdrmw6YpGkGOaOPghVuwoTtqSAkl+zFWIJS89jidvkYG3EqKAkgLKog/TQReCq13
|
||||
HWrF8cMck+Rf2K8k26q/RNZaA9ZUKjLExzz8lsWmd2C7rvkGLrlxnzxz0gGyNR3Q
|
||||
KK74vcPhqeABt2GSkHtEXZFFA9IVVzwlRWK3e0S+mVQnZVjNL+cBPn3/hZHMLesB
|
||||
CucyYZv+DxvT+JkYXBkGSw4s3hpABqGym7gdPUIa0q4rbBFG6xP5sLLBG4yru8vV
|
||||
2dyCMmFqRuxpT49uNfyQ6Vj+dobN6qHnP/9NwfzOixXYBHXR6LBqb/M+iCiJaaIn
|
||||
uiRLHwkQFPrqY0SOHfkQpA//W51vj8meuz7snRO+vZFcjLneFFzqfh1Jdz8IqDpO
|
||||
CkI5pBJmi8e0oSe6r68MkahiQLlYPwm7d+sjHvJhPWipNKWq/uwCgBs+Ac1lpPXR
|
||||
MwLbrZukcLMYlLmb2MrCKmjcMt0BZsZKBNYL3a3X9nHgwXdeqFYS4WQDMCCc09lz
|
||||
9YqfdoEsqRO4qN7D0hFqnwjOzb34ixZ6UO8a8ekY9QKxAgWc9fJWGMg6Pjdg4qsK
|
||||
nqymOIAdGVOJdoRM46wKGVBvbsF2gNfQU4XyzgJo5vHGFwJm6EoSnODlL5e2wsQh
|
||||
uN1oqBt/8ef/plloMEqVBweUBATqSqjRF6IhhYJvWVuQHQL1p1vnV9FebiVj34ir
|
||||
Z8ID+o0AnTJcclbUcDwannGJ0cuDcPhk/v/ahVuoMERCi12qnMBo5B/e6Omyh1yB
|
||||
4pbf4GATGGQipDQG75eC/kP2GQEqJP5WYN0Ar8Le/AA/2xyL7upW0yIByyXCwGEb
|
||||
JRwEgU3+bPyu58bFt8Pftit6J7rA3oBVVMOPrYH5eZwRaj5m2RptwKGL6BfHnhNv
|
||||
ZqmCq9EBGX6L1NI0xHMjEFfXJ8jU01XdfG8nCqkwqsHwslXLhqjJphfHcx89YwbV
|
||||
/15GCuURAv1cKe/7277sOhcvP/QpQqSWgvYExHw8PeFJcTYtF2NrRgNwcQsWS1Rj
|
||||
gXa5Ag0EY+0zcAEQANC5N6kSfezuucAgi+X3BD+MT37mxQyvICSggEJf1LDSmy0+
|
||||
bnvD7setL8CP9etTA2fcVNYKI1oboMyhoCnsRP2jDdv1iXOI/hZg4wSb/D1yUkae
|
||||
fUpxv3Wuci2QKavH2MfraDD7BFMbsQeMcHtn4Rk216T6jndZHnzT1Ih7iX0XeQPb
|
||||
li5fojOiZssgWAVT4HPXFCJB6lI35Hjp35oRYwrtMmu5INinZ79n9h1igGtt1ItZ
|
||||
b7rQKNd772Jxcn4UU71ovORSL/xT5i5sxZ+evQOxkpqUAokMOFaoHcOXLmA1NsFv
|
||||
yryXHK4Ioq9ap2jKlLTWkJWjua9JZ4AmKhbvT8X4ELxIKSCAdJKAWP8ZHbXNu5MD
|
||||
aznyzZQLxSO7uFvu356De75mI5iohZNj5wB5Wju71pBiorTKVj4+iJ4e+xVIzFdG
|
||||
hFC0DehNcl2t9w/y8qHwIQ1yUAjXHLXq0/2jsVeH6bU5q/MsgvUP1jcFe0eyOpxy
|
||||
CDvyFdzZFbI57TnB/fvcZTRZ5ewXMFpH8gzuoFzAjUAP95UjYKgaGdrNPNIy28Ii
|
||||
4zhvdghei2+n9jgiMfcGQg8lyfH5yF0vWWWynX0KcJsRwEZoL2EauVdwq4PcYOoU
|
||||
pQFhpcreCjD4LdZ4yRU4InbhcUogXjrQ9Dz01TbPmQD5b5iso21bCEFBXrhzABEB
|
||||
AAGJAjwEGAEKACYCGwwWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwg
|
||||
hwAKCRAU+upjRI4d+X/XD/sH5xvHPfTJq52v8weFmB52up+DzqG2lyhGdoUQ1Muw
|
||||
dRDLTLXLJrFdfpoOo7/j4Scr0rdc7/dpCn0DLcPuCoPxu+SkjEnVehFmZrGSv7Ga
|
||||
x9dHr3DBh42fdlX/U/EnDuyosY0JU1gNF2/6FIA+bTTOFE3RxfN906RjslYQDjMZ
|
||||
UAlSeLYHOZofdltI0YIr32vrxgdWQGZXPxU4XusDUc0z163OO+TGg7iUNWFZP5Qj
|
||||
ubM7e0YbDX0NPIshk8us99YJmrWnhaix1/W5ryO3DXiGaQ7XFi9u7QofRqvRIctg
|
||||
QXavdepkzJow9V9qpMECAJePIuICq7rm+xy+njjbuF436W7390bfVBwRr+FPADsl
|
||||
jgQP4KvY5rykss30kheom8wNEbveWkhH5oTfH9b7O4KXJfpfJzrlgOWp2BD9JL8t
|
||||
/M4HvFXTr2a75H/QbHK5OFrZeGATuv9OTxv7EZvnrPXU+DYTFldpu7TrNNqKCoj3
|
||||
ZyXmc3Hhg5kskDhfHJppaeOayuhMOpT3ud1MFzROY5SLVIH8rBR12KUgsCUYQcGs
|
||||
Iy0+0QvEGkjb4cAH1NK3VlbqVNsy1RmqRt2B28R2ueewDfTOoqkzt4MmzLqTdnAx
|
||||
mTqmHmkEKhEf3K4MRNUPO2yieUg2COk5l6x9HhAnoxxeOZrTmcMsPY/UViG2HEPm
|
||||
ybkCDQRj7TRDARAA9DZuKdfKq4Bs2+NwxC0aplljWOl8VIsEVg+Q8agD7/HU6/b6
|
||||
Dry0njtWybn2x6Axf/nUdeOC01Fi1lmht/fpj6mRkgAvd/V6P10xnsUoykPSDSTh
|
||||
P25MFFGW3JAA82bwdJ4AJpEQvTZG2nTb3237vlBiI1qHQrac8GYkju2O4UfySRN6
|
||||
7cyi7bMf2pjWBBOEhaNy4b6CMDsb32P/N5J7sTE/TXgrS+u4ITIgjzSrkUkh5Z+B
|
||||
8QVRa7xPIDZJdvZWTEXWu5fgRPZvxbr154GIkWJkFzlDoB1UcO56/uzRUuKhEV6o
|
||||
HW3LMUuWdPMjpHpq8hrL0G2rDniJFUtbDFzHdZK1LUU3T2BJM8rjI3D/euph+IDT
|
||||
27vl5qo72zCYE/iKzx4FMLZcQvx1kUAxkPX8l+dzZEwKeRIIpFDxQvatRtl+z0bM
|
||||
jbkpDb+Yjv66sC4dYRpgTTGX6rok0PWHR3IxDNzyf2j8zQ4LFJ+rVBM1GjGSt6mG
|
||||
j9TeL8CVeiSp4SuJ7I/FJVPHsKb50m+BDzeB31qTydNqh2kKr0DVAUa+TUsCr7e0
|
||||
OYr8WE2adJcRXIW0qw50xXF+W7/05GqSCVD0dpeOUdBTQTsSkQmM3/0hcj9aVo9e
|
||||
UDCM9RF0WRqiDAoHzJFfg+ztamkQI5HO6CklC4Ok22qrHRf6HDNYSuT6QFkAEQEA
|
||||
AYkCPQQYAQoAJwMbIAQWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwf
|
||||
tAAKCRAU+upjRI4d+Y+cD/9yllG6uo934pcHNsVppZBfREFwSc8ywlbosCuSVpay
|
||||
PjSqgrWwDrnqrsk0F2kUdC6rR3BIcXbn+lA9KqylH+cCXAJCkh8EDq6TlQ7Lt5EV
|
||||
w1U0MAMXOyxPwDymQ/BO+iDyjXWkRRYgbF5XiFhCfGeuKyhkhACisAgNZ1uA1P5k
|
||||
0SJYc14YfEhQkB46Y20SpfVHRsQ46FyNB6GHbmTmfoO8La8VTh++7GBdh85HfvkG
|
||||
VNQ3wpi5oXsOLN9+MJOezc0XsW2LQsKQj1/J7QKzGh+lxN5cemsA5aqPzh8dyxeT
|
||||
0lYRFp4AHkimqGUomVpRkbegMIPxXqOE+ZAmsddErw0UtmrKxcmMptOJwNgYzEgu
|
||||
++2vtqerL/NYp+wsdcWaBjCz2F3NiwHgNli7NSB/FPwucZZ5gN5C4SnmeFzrGdHg
|
||||
Oy+tQUN6ayQKljHeBO7CjMlsFNo/dcVrEMa1ShxBMqlj/6ivoEhktLz0Nru4FwNU
|
||||
xE5SJYDYfpjD7Ws8y4LoXgWXjFHrMO6N9GzqLN/e8LT7I+w4ps2MrgJ8QSrelmQ3
|
||||
rjkxp3uWp5v2lqy4rLfpi9iB6zIAeoN2eU1yOM9joxOYMxKYaYeYyP1Mm90wFol8
|
||||
LcTSaN+tVniPddBiL6zvsGBEMbCR9XN3EQ+mErbuw5ovWBOCrr+dvN3FxvD11y4J
|
||||
7w==
|
||||
=mXYP
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,51 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQENBFZu2zwBCADfatenjH3cvhlU6AeInvp4R0JmPBG942aghFj1Qh57smRcO5Bv
|
||||
y9mqrX3UDdmVvu58V3k1k9/GzPnAG1t+c7ohdymv/AMuNY4pE2sfxx7bX+mncTHX
|
||||
5wthipn8kTNm4WjREjCJM1Bm5sozzEZetED3+0/dWlnHl8b38evnLsD+WbSrDPVp
|
||||
o6M6Eg9IfMwTfcXzdmLmSnGolBWDQ9i1a0x0r3o+sDW5UTnr7jVP+zILcnOZ1Ewl
|
||||
Rn9OJ4Qg3ULM7WTMDYpKH4BO7RLR3aJgmsFAHp17vgUnzzFBZ10MCS3UOyUNoyph
|
||||
xo3belf7Q9nrHcSNbqSeQuBnW/vafAZUreAlABEBAAG0IkVlbGNvIERvbHN0cmEg
|
||||
PGVkb2xzdHJhQGdtYWlsLmNvbT6JATwEEwEIACYCGyMHCwkIBwMCAQYVCAIJCgsE
|
||||
FgIDAQIeAQIXgAUCVm7etAIZAQAKCRCBcLRybXGY3q51B/96qt41tmcDSzrj/UTl
|
||||
O6rErfW5zFvVsJTZ95Duwu87t/DVhw5lKBQcjALqVddufw1nMzyN/tSOMVDW8xe4
|
||||
wMEdcU4+QAMzNX80enuyinsw1glxfLcK0+VbTvqNIfw0sG3MjPqNs6cK2VRfMHK4
|
||||
paJjytBVICszNX9TfjLyIpKKoSSo1vqnT47LDZ5GIMy7l9Cs2sO/rqQHSPcR79yz
|
||||
8m8tbHpDDEMZmJeklckKP2QoiqnHiIvlisDxLclYnUmNaPdaN/f++qZz5Yqvu1n+
|
||||
sNUBA5eLaZH64Uy2SwtABxO3JPJ8nQ2+SFZ7ocFm4Gcdv4aM+Ura9S6fvM91tEJp
|
||||
yAQOiQE5BBMBCAAjBQJWbts8AhsjBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AA
|
||||
CgkQgXC0cm1xmN6sIAgAielxO8zJREqEkA2xudg/o4e9ZlNZ3X1NvY8OzJH/qlB2
|
||||
SmwKqwifhtbC1K0uavXA7eaxdtd2zrI+Yq7IooUyv7juMjHTZhLcFbR5iVkQ4Mfp
|
||||
JmeHXJ/ChYKxD5mMj/C3WbCZ91oCSNZ6Iyi5fvQj/691OC4q+y/2NEUcOI8D8cw8
|
||||
XKHbKtceFYc+nZmdOv3ZZrNTSN/kszGViNNLKgnpPdDVPtLp+vjXtbmitiFG2HL/
|
||||
WfbJ+3Gh2Yr1Vy3O9dWKH++e1AmIv7WWqmUjRFVpqC/wr7/BLaScWT8WKF5vkshU
|
||||
gq8Ez1/cuizsgs3wQIZWgXKQK5njvwnbKg+Zmh/uGbQmRWVsY28gRG9sc3RyYSA8
|
||||
ZWVsY28uZG9sc3RyYUB0d2VhZy5pbz6JAU4EEwEIADgWIQS1QdVTAScOC88Vyl2B
|
||||
cLRybXGY3gUCXELt4gIbIwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCBcLRy
|
||||
bXGY3ujFCADfS5D1xHU8KH6TpqgssSggYVq62Wwn/Ga+4XPPetM+ajcXagyH6SwB
|
||||
mxlHICcnv9xC93ryiTI10P1ADJl+aBsI66wEdHBU+ty4RTDy4JZNUPtmRCk9LhSc
|
||||
mtUO3ry/wtWkRLdJxP49hg7BbQvWoU0M6WODp7SJjPKPWNX64mzHBeOuy+DqGCbM
|
||||
lpGNCvW8ahU/ewbm7+xwWmzqLDoWzXjHsdF4QdzMVM/vkAgWEP4y0wEqFASzIYaR
|
||||
GNEkBWU4OQVq5Bdm9+wWWAgsbM0FJAQl0GDqnz4QxWzxxCAAXdbh9F5ffafWYsA9
|
||||
bise4ZQLkvYo6iUnrcFm4dtZbT8iL3gptCtFZWxjbyBEb2xzdHJhIDxlZWxjby5k
|
||||
b2xzdHJhQGxvZ2ljYmxveC5jb20+iQE5BBMBCAAjBQJWbt6nAhsjBwsJCAcDAgEG
|
||||
FQgCCQoLBBYCAwECHgECF4AACgkQgXC0cm1xmN4b/wf8DApMV/jSPEpibekrUPQu
|
||||
Ye3Z8cxBQuRm/nOPowtPEH/ShAevrCdRiob2nuEZWNoqZ2e5/+6ud07Hs9bslvco
|
||||
cDv1jeY1dof1idxfKhH3kfSpuD2XJhuzQBxBqOrIlCS/rdnW+Y9wOGD7+bs9QpcA
|
||||
IyAeQGLLkfggAxaGYQ2Aev8pS7i3a/+lOWbFhcTe02I49KemCOJqBorG5FfILLNr
|
||||
DjO3EoutNGpuz6rZvc/BlymphWBoAdUmxgoObr7NYWgw9pI8WeE6C7bbSOO7p5aQ
|
||||
spWXU7Hm17DkzsVDpaJlyClllqK+DdKza5oWlBMe/P02jD3Y+0P/2rCCyQQwmH3D
|
||||
RbkBDQRWbts8AQgA0g556xc08dH5YNEjbCwEt1j+XoRnV4+GfbSJIXOl9joIgzRC
|
||||
4IaijvL8+4biWvX7HiybfvBKto0XB1AWLZRC3jWKX5p74I77UAcrD+VQ/roWQqlJ
|
||||
BKbiQMlRYEsj/5Xnf72G90IP4DAFKvNl+rLChe+jUySA91BCtrYoP75Sw1BE9Cyz
|
||||
xEtm4WUzKAJdXI+ZTBttA2Nbqy+GSuzBs7fSKDwREJaZmVrosvmns+pQVG4WPWf4
|
||||
0l4mPguDQmZ9wSWZvBDkpG7AgHYDRYRGkMbAGsVfc6cScN2VsSTa6cbeeAEowKxM
|
||||
qx9RbY3WOq6aKAm0qDvow1nl7WwXwe8K0wQxfQARAQABiQEfBBgBCAAJBQJWbts8
|
||||
AhsMAAoJEIFwtHJtcZjeuAAH/0YNz2Qe1IAEO5oqEZNFOccL4KxVPrBhWUen83/b
|
||||
C6PjOnOqv6q5ztAcms88WIKxBlfzIfq+dzJcbKVS/H7TEXgcaC+7EYW8sJVEsipN
|
||||
BtEZ3LQNJ5coDjm7WZygniah1lfXNuiritAXduK5FWNNndqGArEaeZ8Shzdo/Uyi
|
||||
b9lOsBIL6xc2ZcnX5f+rTu02LCEtEb0FwCycZLEWYf8hG4k8uttIOZOC+CLk/k8d
|
||||
kBmPikMwUVTTV0CdT1cemQKdTaoAaK+kurF6FYXwcnjhRlHrisSt/tVMEwTw4LUM
|
||||
3MYf6qfjjvE4HlDwZal8th7ccoQp/flfJIuRv85xCcKK+PI=
|
||||
=u5cX
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -0,0 +1,13 @@
|
||||
# Maintainer GPG Keys
|
||||
|
||||
Release tags are signed by members of the [Nix maintainer team](https://nixos.org/community/teams/nix/) as part of the [release process](../release-process.md). This directory contains the public GPG keys used for signing.
|
||||
|
||||
## Keys
|
||||
|
||||
- **Eelco Dolstra**
|
||||
GPG Fingerprint: `B541 D553 0127 0E0B CF15 CA5D 8170 B472 6D71 98DE`
|
||||
|
||||
- **Sergei Zimmerman**
|
||||
GPG Fingerprint: [`158A 6F53 0EA2 02E5 F651 6113 14FA EA63 448E 1DF9`](https://keys.openpgp.org/vks/v1/by-fingerprint/158A6F530EA202E5F651611314FAEA63448E1DF9)
|
||||
|
||||
<!-- TODO: Add keys for other Nix team members -->
|
||||
@@ -5,11 +5,11 @@
|
||||
The release process is intended to create the following for each
|
||||
release:
|
||||
|
||||
* A Git tag
|
||||
* A signed Git tag (public keys in `maintainers/keys/`)
|
||||
|
||||
* Binary tarballs in https://releases.nixos.org/?prefix=nix/
|
||||
|
||||
* Docker images
|
||||
* Docker images (arm64 and amd64 variants, uploaded to DockerHub and GHCR)
|
||||
|
||||
* Closures in https://cache.nixos.org
|
||||
|
||||
@@ -104,21 +104,17 @@ release:
|
||||
evaluation ID (e.g. `1780832` in
|
||||
`https://hydra.nixos.org/eval/1780832`).
|
||||
|
||||
* Tag the release and upload the release artifacts to
|
||||
[`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/):
|
||||
* Tag the release:
|
||||
|
||||
```console
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
|
||||
```
|
||||
|
||||
Note: `IS_LATEST=1` causes the `latest-release` branch to be
|
||||
force-updated. This is used by the `nixos.org` website to get the
|
||||
[latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/).
|
||||
|
||||
TODO: This script requires the right AWS credentials. Document.
|
||||
|
||||
TODO: This script currently requires a
|
||||
`/home/eelco/Dev/nix-pristine`.
|
||||
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
|
||||
|
||||
TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
|
||||
|
||||
@@ -181,16 +177,18 @@ release:
|
||||
* Wait for the desired evaluation of the maintenance jobset to finish
|
||||
building.
|
||||
|
||||
* Run
|
||||
* Tag the release
|
||||
|
||||
```console
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
|
||||
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
|
||||
```
|
||||
|
||||
Omit `IS_LATEST=1` when creating a point release that is not on the
|
||||
most recent stable branch. This prevents `nixos.org` to going back
|
||||
to an older release.
|
||||
|
||||
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
|
||||
|
||||
* Bump the version number of the release branch as above (e.g. to
|
||||
`2.12.2`).
|
||||
|
||||
|
||||
+128
-82
@@ -1,7 +1,8 @@
|
||||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
|
||||
#! nix-shell -i perl -p awscli2 perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 perlPackages.GetoptLongDescriptive gnupg1
|
||||
|
||||
use strict;
|
||||
use Getopt::Long::Descriptive;
|
||||
use Data::Dumper;
|
||||
use File::Basename;
|
||||
use File::Path;
|
||||
@@ -13,7 +14,30 @@ use Net::Amazon::S3;
|
||||
|
||||
delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
|
||||
|
||||
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
|
||||
my ($opt, $usage) = describe_options(
|
||||
'%c %o <eval-id>',
|
||||
[ 'skip-docker', 'Skip Docker image upload' ],
|
||||
[ 'skip-git', 'Skip Git tagging' ],
|
||||
[ 'skip-s3', 'Skip S3 upload' ],
|
||||
[ 'docker-owner=s', 'Docker image owner', { default => 'nixos/nix' } ],
|
||||
[ 'project-root=s', 'Pristine git repository path' ],
|
||||
[ 's3-endpoint=s', 'Custom S3 endpoint' ],
|
||||
[ 's3-host=s', 'S3 host', { default => 's3-eu-west-1.amazonaws.com' } ],
|
||||
[],
|
||||
[ 'help|h', 'Show this help message', { shortcircuit => 1 } ],
|
||||
[],
|
||||
[ 'Environment variables:' ],
|
||||
[ 'AWS_ACCESS_KEY_ID' ],
|
||||
[ 'AWS_SECRET_ACCESS_KEY' ],
|
||||
[ 'AWS_SESSION_TOKEN For OIDC' ],
|
||||
[ 'IS_LATEST Set to "1" to mark as latest release' ],
|
||||
);
|
||||
|
||||
print($usage->text), exit if $opt->help;
|
||||
|
||||
my $evalId = $ARGV[0] or do { print STDERR $usage->text; exit 1 };
|
||||
|
||||
die "--project-root is required unless --skip-git is specified\n" unless $opt->skip_git || $opt->project_root;
|
||||
|
||||
my $releasesBucketName = "nix-releases";
|
||||
my $channelsBucketName = "nix-channels";
|
||||
@@ -62,25 +86,38 @@ File::Path::make_path($narCache);
|
||||
my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
|
||||
|
||||
# S3 setup.
|
||||
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
|
||||
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
|
||||
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'};
|
||||
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'};
|
||||
my $aws_session_token = $ENV{'AWS_SESSION_TOKEN'};
|
||||
|
||||
my $s3 = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
retry => 1,
|
||||
host => "s3-eu-west-1.amazonaws.com",
|
||||
});
|
||||
my ($s3, $releasesBucket, $s3_channels, $channelsBucket);
|
||||
|
||||
my $releasesBucket = $s3->bucket($releasesBucketName) or die;
|
||||
unless ($opt->skip_s3) {
|
||||
$aws_access_key_id or die "No AWS_ACCESS_KEY_ID given.";
|
||||
$aws_secret_access_key or die "No AWS_SECRET_ACCESS_KEY given.";
|
||||
|
||||
my $s3_us = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
retry => 1,
|
||||
});
|
||||
$s3 = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
|
||||
retry => 1,
|
||||
host => $opt->s3_host,
|
||||
secure => ($opt->s3_endpoint && $opt->s3_endpoint =~ /^http:/) ? 0 : 1,
|
||||
});
|
||||
|
||||
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
|
||||
$releasesBucket = $s3->bucket($releasesBucketName) or die;
|
||||
|
||||
$s3_channels = Net::Amazon::S3->new(
|
||||
{ aws_access_key_id => $aws_access_key_id,
|
||||
aws_secret_access_key => $aws_secret_access_key,
|
||||
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
|
||||
retry => 1,
|
||||
$opt->s3_endpoint ? (host => $opt->s3_host) : (),
|
||||
$opt->s3_endpoint ? (secure => ($opt->s3_endpoint =~ /^http:/) ? 0 : 1) : (),
|
||||
});
|
||||
|
||||
$channelsBucket = $s3_channels->bucket($channelsBucketName) or die;
|
||||
}
|
||||
|
||||
sub getStorePath {
|
||||
my ($jobName, $output) = @_;
|
||||
@@ -115,11 +152,12 @@ sub copyManual {
|
||||
File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1});
|
||||
}
|
||||
|
||||
system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
|
||||
my $awsEndpoint = $opt->s3_endpoint ? "--endpoint-url " . $opt->s3_endpoint : "";
|
||||
system("aws $awsEndpoint s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
|
||||
or die "syncing manual to S3\n";
|
||||
}
|
||||
|
||||
copyManual;
|
||||
copyManual unless $opt->skip_s3;
|
||||
|
||||
sub downloadFile {
|
||||
my ($jobName, $productNr, $dstName) = @_;
|
||||
@@ -158,30 +196,12 @@ sub downloadFile {
|
||||
return $sha256_expected;
|
||||
}
|
||||
|
||||
downloadFile("binaryTarball.i686-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-linux", "1");
|
||||
downloadFile("binaryTarball.aarch64-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-darwin", "1");
|
||||
downloadFile("binaryTarball.aarch64-darwin", "1");
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
downloadFile("installerScript", "1");
|
||||
|
||||
# Upload docker images to dockerhub.
|
||||
# Upload docker images.
|
||||
my $dockerManifest = "";
|
||||
my $dockerManifestLatest = "";
|
||||
my $haveDocker = 0;
|
||||
|
||||
unless ($opt->skip_docker) {
|
||||
for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
|
||||
my $system = $platforms->[0];
|
||||
my $dockerPlatform = $platforms->[1];
|
||||
@@ -195,8 +215,8 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
|
||||
print STDERR "loading docker image for $dockerPlatform...\n";
|
||||
system("docker load -i $tmpDir/$fn") == 0 or die;
|
||||
|
||||
my $tag = "nixos/nix:$version-$dockerPlatform";
|
||||
my $latestTag = "nixos/nix:latest-$dockerPlatform";
|
||||
my $tag = $opt->docker_owner . ":$version-$dockerPlatform";
|
||||
my $latestTag = $opt->docker_owner . ":latest-$dockerPlatform";
|
||||
|
||||
print STDERR "tagging $version docker image for $dockerPlatform...\n";
|
||||
system("docker tag nix:$version $tag") == 0 or die;
|
||||
@@ -219,68 +239,94 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
|
||||
}
|
||||
|
||||
if ($haveDocker) {
|
||||
my $dockerOwner = $opt->docker_owner;
|
||||
print STDERR "creating multi-platform docker manifest...\n";
|
||||
system("docker manifest rm nixos/nix:$version");
|
||||
system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
|
||||
system("docker manifest rm $dockerOwner:$version");
|
||||
system("docker manifest create $dockerOwner:$version $dockerManifest") == 0 or die;
|
||||
if ($isLatest) {
|
||||
print STDERR "creating latest multi-platform docker manifest...\n";
|
||||
system("docker manifest rm nixos/nix:latest");
|
||||
system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
|
||||
system("docker manifest rm $dockerOwner:latest");
|
||||
system("docker manifest create $dockerOwner:latest $dockerManifestLatest") == 0 or die;
|
||||
}
|
||||
|
||||
print STDERR "pushing multi-platform docker manifest...\n";
|
||||
system("docker manifest push nixos/nix:$version") == 0 or die;
|
||||
system("docker manifest push $dockerOwner:$version") == 0 or die;
|
||||
|
||||
if ($isLatest) {
|
||||
print STDERR "pushing latest multi-platform docker manifest...\n";
|
||||
system("docker manifest push nixos/nix:latest") == 0 or die;
|
||||
system("docker manifest push $dockerOwner:latest") == 0 or die;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Upload nix-fallback-paths.nix.
|
||||
write_file("$tmpDir/fallback-paths.nix",
|
||||
"{\n" .
|
||||
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
|
||||
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
|
||||
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
|
||||
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
|
||||
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
|
||||
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
|
||||
"}\n");
|
||||
|
||||
# Upload release files to S3.
|
||||
for my $fn (glob "$tmpDir/*") {
|
||||
my $name = basename($fn);
|
||||
next if $name eq "manual";
|
||||
my $dstKey = "$releaseDir/" . $name;
|
||||
unless (defined $releasesBucket->head_key($dstKey)) {
|
||||
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
|
||||
unless ($opt->skip_s3) {
|
||||
downloadFile("binaryTarball.i686-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-linux", "1");
|
||||
downloadFile("binaryTarball.aarch64-linux", "1");
|
||||
downloadFile("binaryTarball.x86_64-darwin", "1");
|
||||
downloadFile("binaryTarball.aarch64-darwin", "1");
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
eval {
|
||||
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
|
||||
};
|
||||
warn "$@" if $@;
|
||||
downloadFile("installerScript", "1");
|
||||
|
||||
my $configuration = ();
|
||||
$configuration->{content_type} = "application/octet-stream";
|
||||
# Upload nix-fallback-paths.nix.
|
||||
write_file("$tmpDir/fallback-paths.nix",
|
||||
"{\n" .
|
||||
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
|
||||
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
|
||||
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
|
||||
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
|
||||
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
|
||||
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
|
||||
"}\n");
|
||||
|
||||
if ($fn =~ /.sha256|install|\.nix$/) {
|
||||
$configuration->{content_type} = "text/plain";
|
||||
for my $fn (glob "$tmpDir/*") {
|
||||
my $name = basename($fn);
|
||||
next if $name eq "manual";
|
||||
my $dstKey = "$releaseDir/" . $name;
|
||||
unless (defined $releasesBucket->head_key($dstKey)) {
|
||||
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
|
||||
|
||||
my $configuration = ();
|
||||
$configuration->{content_type} = "application/octet-stream";
|
||||
|
||||
if ($fn =~ /.sha256|install|\.nix$/) {
|
||||
$configuration->{content_type} = "text/plain";
|
||||
}
|
||||
|
||||
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
|
||||
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
|
||||
}
|
||||
|
||||
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
|
||||
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
|
||||
}
|
||||
|
||||
# Update the "latest" symlink.
|
||||
$channelsBucket->add_key(
|
||||
"nix-latest/install", "",
|
||||
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
|
||||
or die $channelsBucket->err . ": " . $channelsBucket->errstr
|
||||
if $isLatest;
|
||||
}
|
||||
|
||||
# Update the "latest" symlink.
|
||||
$channelsBucket->add_key(
|
||||
"nix-latest/install", "",
|
||||
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
|
||||
or die $channelsBucket->err . ": " . $channelsBucket->errstr
|
||||
if $isLatest;
|
||||
|
||||
# Tag the release in Git.
|
||||
chdir("/home/eelco/Dev/nix-pristine") or die;
|
||||
system("git remote update origin") == 0 or die;
|
||||
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
|
||||
system("git push --tags") == 0 or die;
|
||||
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
|
||||
unless ($opt->skip_git) {
|
||||
chdir($opt->project_root) or die "Cannot chdir to " . $opt->project_root . ": $!";
|
||||
system("git remote update origin") == 0 or die;
|
||||
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
|
||||
system("git push origin refs/tags/$version") == 0 or die;
|
||||
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
|
||||
}
|
||||
|
||||
File::Path::remove_tree($narCache, {safe => 1});
|
||||
File::Path::remove_tree($tmpDir, {safe => 1});
|
||||
@@ -63,6 +63,3 @@ subproject('nix-functional-tests')
|
||||
if get_option('json-schema-checks')
|
||||
subproject('json-schema-checks')
|
||||
endif
|
||||
if get_option('kaitai-struct-checks')
|
||||
subproject('kaitai-struct-checks')
|
||||
endif
|
||||
@@ -28,13 +28,6 @@ option(
|
||||
description : 'Build benchmarks (requires gbenchmark)',
|
||||
)
|
||||
|
||||
option(
|
||||
'kaitai-struct-checks',
|
||||
type : 'boolean',
|
||||
value : true,
|
||||
description : 'Check the Kaitai Struct specifications (requires Kaitai Struct)',
|
||||
)
|
||||
|
||||
option(
|
||||
'json-schema-checks',
|
||||
type : 'boolean',
|
||||
|
||||
@@ -11,7 +11,7 @@ ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
|
||||
KillMode=process
|
||||
LimitNOFILE=1048576
|
||||
TasksMax=1048576
|
||||
Delegate=yes
|
||||
Delegate=
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -133,7 +133,7 @@ let
|
||||
+
|
||||
lib.optionalString
|
||||
(
|
||||
!stdenv.hostPlatform.isWindows
|
||||
!(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin)
|
||||
# build failure
|
||||
&& !stdenv.hostPlatform.isStatic
|
||||
# LTO breaks exception handling on x86-64-darwin.
|
||||
@@ -452,11 +452,6 @@ in
|
||||
*/
|
||||
nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { };
|
||||
|
||||
/**
|
||||
Kaitai struct schema validation checks
|
||||
*/
|
||||
nix-kaitai-struct-checks = callPackage ../src/kaitai-struct-checks/package.nix { };
|
||||
|
||||
nix-perl-bindings = callPackage ../src/perl/package.nix { };
|
||||
|
||||
/**
|
||||
|
||||
@@ -40,6 +40,7 @@ scope: {
|
||||
postInstall =
|
||||
lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
|
||||
(prevAttrs.postInstall or "");
|
||||
patches = [ ];
|
||||
});
|
||||
|
||||
# TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
|
||||
|
||||
+54
-40
@@ -142,12 +142,23 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
internalDrvs = byDrvPath (
|
||||
# Drop the attr names (not present in buildInputs anyway)
|
||||
lib.attrValues availableComponents
|
||||
++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents)
|
||||
++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) (
|
||||
lib.attrValues availableComponents
|
||||
)
|
||||
);
|
||||
|
||||
isInternal =
|
||||
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
|
||||
|
||||
activeComponentNames = lib.listToAttrs (
|
||||
map (c: {
|
||||
name = c.pname or c.name;
|
||||
value = null;
|
||||
}) activeComponents
|
||||
);
|
||||
|
||||
isActiveComponent = name: activeComponentNames ? ${name};
|
||||
|
||||
in
|
||||
{
|
||||
pname = "shell-for-nix";
|
||||
@@ -190,27 +201,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
}
|
||||
);
|
||||
|
||||
small =
|
||||
(finalAttrs.finalPackage.withActiveComponents (
|
||||
c:
|
||||
lib.intersectAttrs (lib.genAttrs [
|
||||
"nix-cli"
|
||||
"nix-util-tests"
|
||||
"nix-store-tests"
|
||||
"nix-expr-tests"
|
||||
"nix-fetchers-tests"
|
||||
"nix-flake-tests"
|
||||
"nix-functional-tests"
|
||||
"nix-perl-bindings"
|
||||
] (_: null)) c
|
||||
)).overrideAttrs
|
||||
(o: {
|
||||
mesonFlags = o.mesonFlags ++ [
|
||||
# TODO: infer from activeComponents or vice versa
|
||||
"-Dkaitai-struct-checks=false"
|
||||
"-Djson-schema-checks=false"
|
||||
];
|
||||
});
|
||||
small = finalAttrs.finalPackage.withActiveComponents (
|
||||
c:
|
||||
lib.intersectAttrs (lib.genAttrs [
|
||||
"nix-cli"
|
||||
"nix-util-tests"
|
||||
"nix-store-tests"
|
||||
"nix-expr-tests"
|
||||
"nix-fetchers-tests"
|
||||
"nix-flake-tests"
|
||||
"nix-functional-tests"
|
||||
"nix-perl-bindings"
|
||||
] (_: null)) c
|
||||
);
|
||||
};
|
||||
|
||||
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
|
||||
@@ -278,21 +281,32 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
|
||||
dontUseCmakeConfigure = true;
|
||||
|
||||
mesonFlags =
|
||||
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
|
||||
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
|
||||
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
|
||||
++ lib.optionals havePerl (
|
||||
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
|
||||
)
|
||||
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
|
||||
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
|
||||
mesonFlags = [
|
||||
(lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
|
||||
]
|
||||
++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
|
||||
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
|
||||
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
|
||||
++ lib.optionals havePerl (
|
||||
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
|
||||
)
|
||||
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
|
||||
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
|
||||
|
||||
nativeBuildInputs =
|
||||
let
|
||||
inputs =
|
||||
dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
|
||||
lib.filter (x: !isInternal x) (
|
||||
lib.lists.concatMap (
|
||||
# Nix manual has a build-time dependency on nix, but we
|
||||
# don't want to do a native build just to enter the cross
|
||||
# dev shell.
|
||||
#
|
||||
# TODO: think of a more principled fix for this.
|
||||
c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs
|
||||
) activeComponents
|
||||
)
|
||||
)
|
||||
++ lib.optional (
|
||||
!buildCanExecuteHost
|
||||
@@ -308,8 +322,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
pkgs.buildPackages.nixfmt-rfc-style
|
||||
pkgs.buildPackages.shellcheck
|
||||
pkgs.buildPackages.include-what-you-use
|
||||
pkgs.buildPackages.gdb
|
||||
]
|
||||
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
|
||||
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
|
||||
lib.hiPrio pkgs.buildPackages.clang-tools
|
||||
)
|
||||
@@ -325,13 +339,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
)
|
||||
);
|
||||
|
||||
buildInputs = [
|
||||
pkgs.gbenchmark
|
||||
]
|
||||
++ dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
||||
)
|
||||
++ lib.optional havePerl pkgs.perl;
|
||||
buildInputs =
|
||||
# TODO change Nixpkgs to mark gbenchmark as building on Windows
|
||||
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
|
||||
++ dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
||||
)
|
||||
++ lib.optional havePerl pkgs.perl;
|
||||
|
||||
propagatedBuildInputs = dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
|
||||
|
||||
@@ -72,7 +72,6 @@ let
|
||||
"nix-manual-manpages-only"
|
||||
"nix-internal-api-docs"
|
||||
"nix-external-api-docs"
|
||||
"nix-kaitai-struct-checks"
|
||||
]
|
||||
);
|
||||
in
|
||||
|
||||
@@ -53,8 +53,8 @@ readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d
|
||||
|
||||
readonly NIX_INSTALLED_NIX="@nix@"
|
||||
readonly NIX_INSTALLED_CACERT="@cacert@"
|
||||
#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
|
||||
#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
|
||||
#readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6"
|
||||
#readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2"
|
||||
EXTRACTED_NIX_PATH="$(dirname "$0")"
|
||||
readonly EXTRACTED_NIX_PATH
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../../.version
|
||||
@@ -1,77 +0,0 @@
|
||||
# Run with:
|
||||
# meson test --suite kaitai-struct
|
||||
# Run with: (without shell / configure)
|
||||
# nix build .#nix-kaitai-struct-checks
|
||||
|
||||
project(
|
||||
'nix-kaitai-struct-checks',
|
||||
'cpp',
|
||||
version : files('.version'),
|
||||
default_options : [
|
||||
'cpp_std=c++23',
|
||||
# TODO(Qyriad): increase the warning level
|
||||
'warning_level=1',
|
||||
'errorlogs=true', # Please print logs for tests that fail
|
||||
],
|
||||
meson_version : '>= 1.1',
|
||||
license : 'LGPL-2.1-or-later',
|
||||
)
|
||||
|
||||
kaitai_runtime_dep = dependency('kaitai-struct-cpp-stl-runtime', required : true)
|
||||
gtest_dep = dependency('gtest')
|
||||
gtest_main_dep = dependency('gtest_main', required : true)
|
||||
|
||||
# Find the Kaitai Struct compiler
|
||||
ksc = find_program('ksc', required : true)
|
||||
|
||||
kaitai_generated_srcs = custom_target(
|
||||
'kaitai-generated-sources',
|
||||
input : [ 'nar.ksy' ],
|
||||
output : [ 'nix_nar.cpp', 'nix_nar.h' ],
|
||||
command : [
|
||||
ksc,
|
||||
'@INPUT@',
|
||||
'--target', 'cpp_stl',
|
||||
'--outdir',
|
||||
meson.current_build_dir(),
|
||||
],
|
||||
)
|
||||
|
||||
nar_kaitai_lib = library(
|
||||
'nix-nar-kaitai-lib',
|
||||
kaitai_generated_srcs,
|
||||
dependencies : [ kaitai_runtime_dep ],
|
||||
install : true,
|
||||
)
|
||||
|
||||
nar_kaitai_dep = declare_dependency(
|
||||
link_with : nar_kaitai_lib,
|
||||
sources : kaitai_generated_srcs[1],
|
||||
)
|
||||
|
||||
# The nar directory is a committed symlink to the actual nars location
|
||||
nars_dir = meson.current_source_dir() / 'nars'
|
||||
|
||||
# Get all example files
|
||||
nars = [
|
||||
'dot.nar',
|
||||
]
|
||||
|
||||
test_deps = [
|
||||
nar_kaitai_dep,
|
||||
kaitai_runtime_dep,
|
||||
gtest_main_dep,
|
||||
]
|
||||
|
||||
this_exe = executable(
|
||||
meson.project_name(),
|
||||
'test-parse-nar.cc',
|
||||
dependencies : test_deps,
|
||||
)
|
||||
|
||||
test(
|
||||
meson.project_name(),
|
||||
this_exe,
|
||||
env : [ 'NIX_NARS_DIR=' + nars_dir ],
|
||||
protocol : 'gtest',
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
../../doc/manual/source/protocols/nix-archive/nar.ksy
|
||||
@@ -1 +0,0 @@
|
||||
../libutil-tests/data/nars
|
||||
@@ -1 +0,0 @@
|
||||
../../nix-meson-build-support
|
||||
@@ -1,70 +0,0 @@
|
||||
# Run with: nix build .#nix-kaitai-struct-checks
|
||||
# or: `nix develop .#nix-kaitai-struct-checks` to enter a dev shell
|
||||
{
|
||||
lib,
|
||||
mkMesonDerivation,
|
||||
gtest,
|
||||
meson,
|
||||
ninja,
|
||||
pkg-config,
|
||||
kaitai-struct-compiler,
|
||||
fetchzip,
|
||||
kaitai-struct-cpp-stl-runtime,
|
||||
# Configuration Options
|
||||
version,
|
||||
}:
|
||||
let
|
||||
inherit (lib) fileset;
|
||||
in
|
||||
mkMesonDerivation (finalAttrs: {
|
||||
pname = "nix-kaitai-struct-checks";
|
||||
inherit version;
|
||||
|
||||
workDir = ./.;
|
||||
fileset = lib.fileset.unions [
|
||||
../../nix-meson-build-support
|
||||
./nix-meson-build-support
|
||||
./.version
|
||||
../../.version
|
||||
../../doc/manual/source/protocols/nix-archive/nar.ksy
|
||||
./nars
|
||||
../../src/libutil-tests/data
|
||||
./meson.build
|
||||
./nar.ksy
|
||||
(fileset.fileFilter (file: file.hasExt "cc") ./.)
|
||||
(fileset.fileFilter (file: file.hasExt "hh") ./.)
|
||||
];
|
||||
|
||||
outputs = [ "out" ];
|
||||
|
||||
buildInputs = [
|
||||
gtest
|
||||
kaitai-struct-cpp-stl-runtime
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
meson
|
||||
ninja
|
||||
pkg-config
|
||||
# This can go away when we bump up to 25.11
|
||||
(kaitai-struct-compiler.overrideAttrs (finalAttrs: {
|
||||
version = "0.11";
|
||||
src = fetchzip {
|
||||
url = "https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/${version}/kaitai-struct-compiler-${version}.zip";
|
||||
sha256 = "sha256-j9TEilijqgIiD0GbJfGKkU1FLio9aTopIi1v8QT1b+A=";
|
||||
};
|
||||
}))
|
||||
];
|
||||
|
||||
doCheck = true;
|
||||
|
||||
mesonCheckFlags = [ "--print-errorlogs" ];
|
||||
|
||||
postInstall = ''
|
||||
touch $out
|
||||
'';
|
||||
|
||||
meta = {
|
||||
platforms = lib.platforms.unix;
|
||||
};
|
||||
})
|
||||
@@ -1,48 +0,0 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include <kaitai/kaitaistream.h>
|
||||
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "nix_nar.h"
|
||||
|
||||
static const std::vector<std::string> NarFiles = {
|
||||
"empty.nar",
|
||||
"dot.nar",
|
||||
"dotdot.nar",
|
||||
"executable-after-contents.nar",
|
||||
"invalid-tag-instead-of-contents.nar",
|
||||
"name-after-node.nar",
|
||||
"nul-character.nar",
|
||||
"slash.nar",
|
||||
};
|
||||
|
||||
class NarParseTest : public ::testing::TestWithParam<std::string>
|
||||
{};
|
||||
|
||||
TEST_P(NarParseTest, ParseSucceeds)
|
||||
{
|
||||
const auto nar_file = GetParam();
|
||||
|
||||
const char * nars_dir_env = std::getenv("NIX_NARS_DIR");
|
||||
if (nars_dir_env == nullptr) {
|
||||
FAIL() << "NIX_NARS_DIR environment variable not set.";
|
||||
}
|
||||
|
||||
const std::filesystem::path nar_file_path = std::filesystem::path(nars_dir_env) / "dot.nar";
|
||||
ASSERT_TRUE(std::filesystem::exists(nar_file_path)) << "Missing test file: " << nar_file_path;
|
||||
|
||||
std::ifstream ifs(nar_file_path, std::ifstream::binary);
|
||||
ASSERT_TRUE(ifs.is_open()) << "Failed to open file: " << nar_file;
|
||||
kaitai::kstream ks(&ifs);
|
||||
nix_nar_t nar(&ks);
|
||||
ASSERT_TRUE(nar.root_node() != nullptr) << "Failed to parse NAR file: " << nar_file;
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(AllNarFiles, NarParseTest, ::testing::ValuesIn(NarFiles));
|
||||
@@ -300,6 +300,8 @@ MixDefaultProfile::MixDefaultProfile()
|
||||
profile = getDefaultProfile().string();
|
||||
}
|
||||
|
||||
static constexpr auto environmentVariablesCategory = "Options that change environment variables";
|
||||
|
||||
MixEnvironment::MixEnvironment()
|
||||
: ignoreEnvironment(false)
|
||||
{
|
||||
|
||||
@@ -116,7 +116,11 @@ MixFlakeOptions::MixFlakeOptions()
|
||||
.labels = {"input-path"},
|
||||
.handler = {[&](std::string s) {
|
||||
warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
|
||||
lockFlags.inputUpdates.insert(flake::parseInputAttrPath(s));
|
||||
auto path = flake::NonEmptyInputAttrPath::parse(s);
|
||||
if (!path)
|
||||
throw UsageError(
|
||||
"--update-input was passed a zero-length input path, which would refer to the flake itself, not an input");
|
||||
lockFlags.inputUpdates.insert(*path);
|
||||
}},
|
||||
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
|
||||
completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
|
||||
@@ -125,14 +129,18 @@ MixFlakeOptions::MixFlakeOptions()
|
||||
|
||||
addFlag({
|
||||
.longName = "override-input",
|
||||
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
|
||||
.description =
|
||||
"Override a specific flake input (e.g. `dwarffs/nixpkgs`). The input path must not be empty. This implies `--no-write-lock-file`.",
|
||||
.category = category,
|
||||
.labels = {"input-path", "flake-url"},
|
||||
.handler = {[&](std::string inputAttrPath, std::string flakeRef) {
|
||||
lockFlags.writeLockFile = false;
|
||||
auto path = flake::NonEmptyInputAttrPath::parse(inputAttrPath);
|
||||
if (!path)
|
||||
throw UsageError(
|
||||
"--override-input was passed a zero-length input path, which would refer to the flake itself, not an input");
|
||||
lockFlags.inputOverrides.insert_or_assign(
|
||||
flake::parseInputAttrPath(inputAttrPath),
|
||||
parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true));
|
||||
std::move(*path), parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true));
|
||||
}},
|
||||
.completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
|
||||
if (n == 0) {
|
||||
|
||||
+1
-1
@@ -795,7 +795,7 @@ void NixRepl::loadFiles()
|
||||
loadedFiles.clear();
|
||||
|
||||
for (auto & i : old) {
|
||||
notice("Loading '%1%'...", i);
|
||||
notice("Loading %1%...", PathFmt(i));
|
||||
loadFile(i);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "nix/expr/eval-gc.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
nix::initLibStore(false);
|
||||
nix::initGC();
|
||||
|
||||
::benchmark::Initialize(&argc, argv);
|
||||
::benchmark::RunSpecifiedBenchmarks();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "nix/expr/eval.hh"
|
||||
#include "nix/expr/eval-settings.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
static std::string mkDynamicAttrsExpr(size_t attrCount)
|
||||
{
|
||||
std::string res;
|
||||
res.reserve(attrCount * 24);
|
||||
res += "{ ";
|
||||
for (size_t i = 0; i < attrCount; ++i) {
|
||||
res += "${\"a";
|
||||
res += std::to_string(i);
|
||||
res += "\"} = ";
|
||||
res += std::to_string(i);
|
||||
res += "; ";
|
||||
}
|
||||
res += "}";
|
||||
return res;
|
||||
}
|
||||
|
||||
static void BM_EvalDynamicAttrs(benchmark::State & state)
|
||||
{
|
||||
const auto attrCount = static_cast<size_t>(state.range(0));
|
||||
const auto exprStr = mkDynamicAttrsExpr(attrCount);
|
||||
|
||||
for (auto _ : state) {
|
||||
state.PauseTiming();
|
||||
|
||||
auto store = openStore("dummy://");
|
||||
fetchers::Settings fetchSettings{};
|
||||
bool readOnlyMode = true;
|
||||
EvalSettings evalSettings{readOnlyMode};
|
||||
evalSettings.nixPath = {};
|
||||
|
||||
EvalState st({}, store, fetchSettings, evalSettings, nullptr);
|
||||
Expr * expr = st.parseExprFromString(exprStr, st.rootPath(CanonPath::root));
|
||||
|
||||
Value v;
|
||||
|
||||
state.ResumeTiming();
|
||||
|
||||
st.eval(expr, v);
|
||||
st.forceValue(v, noPos);
|
||||
benchmark::DoNotOptimize(v);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * attrCount);
|
||||
}
|
||||
|
||||
BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000);
|
||||
@@ -0,0 +1,64 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "nix/expr/get-drvs.hh"
|
||||
#include "nix/expr/eval-settings.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/util/fmt.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
namespace {
|
||||
|
||||
struct GetDerivationsEnv
|
||||
{
|
||||
ref<Store> store = openStore("dummy://");
|
||||
fetchers::Settings fetchSettings{};
|
||||
bool readOnlyMode = true;
|
||||
EvalSettings evalSettings{readOnlyMode};
|
||||
EvalState state;
|
||||
|
||||
Bindings * autoArgs = nullptr;
|
||||
Value attrsValue;
|
||||
|
||||
explicit GetDerivationsEnv(size_t attrCount)
|
||||
: evalSettings([&]() {
|
||||
EvalSettings settings{readOnlyMode};
|
||||
settings.nixPath = {};
|
||||
return settings;
|
||||
}())
|
||||
, state({}, store, fetchSettings, evalSettings, nullptr)
|
||||
{
|
||||
autoArgs = state.buildBindings(0).finish();
|
||||
|
||||
auto attrs = state.buildBindings(attrCount);
|
||||
|
||||
for (size_t i = 0; i < attrCount; ++i) {
|
||||
auto name = fmt("pkg%|1$06d|", i);
|
||||
auto sym = state.symbols.create(name);
|
||||
auto & v = attrs.alloc(sym);
|
||||
v.mkInt(i);
|
||||
}
|
||||
|
||||
attrsValue.mkAttrs(attrs.finish());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
static void BM_GetDerivationsAttrScan(benchmark::State & state)
|
||||
{
|
||||
const auto attrCount = static_cast<size_t>(state.range(0));
|
||||
GetDerivationsEnv env(attrCount);
|
||||
|
||||
for (auto _ : state) {
|
||||
PackageInfos drvs;
|
||||
getDerivations(
|
||||
env.state, env.attrsValue, /*pathPrefix=*/"", *env.autoArgs, drvs, /*ignoreAssertionFailures=*/true);
|
||||
benchmark::DoNotOptimize(drvs.size());
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * attrCount);
|
||||
}
|
||||
|
||||
BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000);
|
||||
@@ -87,3 +87,33 @@ test(
|
||||
},
|
||||
protocol : 'gtest',
|
||||
)
|
||||
|
||||
# Build benchmarks if enabled
|
||||
if get_option('benchmarks')
|
||||
gbenchmark = dependency('benchmark', required : true)
|
||||
|
||||
benchmark_sources = files(
|
||||
'bench-main.cc',
|
||||
'dynamic-attrs-bench.cc',
|
||||
'get-drvs-bench.cc',
|
||||
'regex-cache-bench.cc',
|
||||
)
|
||||
|
||||
benchmark_exe = executable(
|
||||
'nix-expr-benchmarks',
|
||||
benchmark_sources,
|
||||
config_priv_h,
|
||||
dependencies : deps_private_subproject + deps_private + deps_other + [
|
||||
gbenchmark,
|
||||
],
|
||||
include_directories : include_dirs,
|
||||
link_args : linker_export_flags,
|
||||
install : true,
|
||||
cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [],
|
||||
)
|
||||
|
||||
benchmark(
|
||||
'nix-expr-benchmarks',
|
||||
benchmark_exe,
|
||||
)
|
||||
endif
|
||||
@@ -0,0 +1,9 @@
|
||||
# vim: filetype=meson
|
||||
|
||||
option(
|
||||
'benchmarks',
|
||||
type : 'boolean',
|
||||
value : false,
|
||||
description : 'Build benchmarks (requires gbenchmark)',
|
||||
yield : true,
|
||||
)
|
||||
@@ -33,7 +33,7 @@ mkMesonExecutable (finalAttrs: {
|
||||
../../.version
|
||||
./.version
|
||||
./meson.build
|
||||
# ./meson.options
|
||||
./meson.options
|
||||
(fileset.fileFilter (file: file.hasExt "cc") ./.)
|
||||
(fileset.fileFilter (file: file.hasExt "hh") ./.)
|
||||
];
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "nix/expr/eval.hh"
|
||||
#include "nix/expr/eval-settings.hh"
|
||||
#include "nix/fetchers/fetch-settings.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state)
|
||||
{
|
||||
static constexpr int iterations = 5'000;
|
||||
|
||||
static constexpr std::string_view exprStr =
|
||||
"builtins.foldl' "
|
||||
"(acc: _: acc + builtins.length (builtins.match \"a\" \"a\")) "
|
||||
"0 "
|
||||
"(builtins.genList (x: x) "
|
||||
"5000)";
|
||||
|
||||
for (auto _ : state) {
|
||||
state.PauseTiming();
|
||||
|
||||
auto store = openStore("dummy://");
|
||||
fetchers::Settings fetchSettings{};
|
||||
bool readOnlyMode = true;
|
||||
EvalSettings evalSettings{readOnlyMode};
|
||||
evalSettings.nixPath = {};
|
||||
|
||||
EvalState st({}, store, fetchSettings, evalSettings, nullptr);
|
||||
Expr * expr = st.parseExprFromString(std::string(exprStr), st.rootPath(CanonPath::root));
|
||||
|
||||
Value v;
|
||||
|
||||
state.ResumeTiming();
|
||||
|
||||
st.eval(expr, v);
|
||||
st.forceValue(v, noPos);
|
||||
benchmark::DoNotOptimize(v);
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * iterations);
|
||||
}
|
||||
|
||||
BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex);
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "nix/expr/attr-path.hh"
|
||||
#include "nix/expr/eval-inline.hh"
|
||||
#include "nix/util/util.hh"
|
||||
#include "nix/util/strings-inline.hh"
|
||||
|
||||
namespace nix {
|
||||
@@ -137,16 +138,15 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
|
||||
|
||||
auto fail = [fn]() { throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); };
|
||||
|
||||
try {
|
||||
auto colon = fn.rfind(':');
|
||||
if (colon == std::string::npos)
|
||||
fail();
|
||||
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
|
||||
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
|
||||
} catch (std::invalid_argument & e) {
|
||||
auto colon = fn.rfind(':');
|
||||
if (colon == std::string::npos)
|
||||
fail();
|
||||
unreachable();
|
||||
}
|
||||
|
||||
auto lineno = string2Int<uint32_t>(std::string_view(fn).substr(colon + 1));
|
||||
if (!lineno)
|
||||
fail();
|
||||
|
||||
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, *lineno};
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -75,7 +75,7 @@ struct AttrDb
|
||||
|
||||
auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite");
|
||||
|
||||
state->db = SQLite(dbPath);
|
||||
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
@@ -265,7 +265,7 @@ struct AttrDb
|
||||
case AttrType::String: {
|
||||
NixStringContext context;
|
||||
if (!queryAttribute.isNull(3))
|
||||
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
|
||||
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), " "))
|
||||
context.insert(NixStringContextElem::parse(s));
|
||||
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
|
||||
}
|
||||
|
||||
@@ -111,6 +111,7 @@ template class EvalErrorBuilder<TypeError>;
|
||||
template class EvalErrorBuilder<UndefinedVarError>;
|
||||
template class EvalErrorBuilder<MissingArgumentError>;
|
||||
template class EvalErrorBuilder<InfiniteRecursionError>;
|
||||
template class EvalErrorBuilder<StackOverflowError>;
|
||||
template class EvalErrorBuilder<InvalidPathError>;
|
||||
template class EvalErrorBuilder<IFDError>;
|
||||
|
||||
|
||||
@@ -139,7 +139,7 @@ public:
|
||||
, profileFd([&]() {
|
||||
AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
|
||||
if (!fd)
|
||||
throw SysError("opening file %s", profileFile);
|
||||
throw SysError("opening file %s", PathFmt(profileFile));
|
||||
return fd;
|
||||
}())
|
||||
, posCache(state)
|
||||
|
||||
@@ -2408,6 +2408,8 @@ BackedStringView EvalState::coerceToString(
|
||||
bool copyToStore,
|
||||
bool canonicalizePath)
|
||||
{
|
||||
auto _level = addCallDepth(pos);
|
||||
|
||||
forceValue(v, pos);
|
||||
|
||||
if (v.type() == nString) {
|
||||
@@ -2627,6 +2629,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
|
||||
// `assert a == b; x` are critical for our users' testing UX.
|
||||
void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
|
||||
{
|
||||
auto _level = addCallDepth(pos);
|
||||
|
||||
// This implementation must match eqValues.
|
||||
forceValue(v1, pos);
|
||||
forceValue(v2, pos);
|
||||
@@ -2833,6 +2837,8 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
|
||||
// This implementation must match assertEqValues
|
||||
bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
|
||||
{
|
||||
auto _level = addCallDepth(pos);
|
||||
|
||||
forceValue(v1, pos);
|
||||
forceValue(v2, pos);
|
||||
|
||||
|
||||
+25
-2
@@ -213,6 +213,8 @@ StringSet PackageInfo::queryMetaNames()
|
||||
|
||||
bool PackageInfo::checkMeta(Value & v)
|
||||
{
|
||||
auto _level = state->addCallDepth(v.determinePos(noPos));
|
||||
|
||||
state->forceValue(v, v.determinePos(noPos));
|
||||
if (v.type() == nList) {
|
||||
for (auto elem : v.listView())
|
||||
@@ -367,7 +369,26 @@ static std::string addToPath(const std::string & s1, std::string_view s2)
|
||||
return s1.empty() ? std::string(s2) : s1 + "." + s2;
|
||||
}
|
||||
|
||||
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
|
||||
static bool isAttrPathComponent(std::string_view symbol)
|
||||
{
|
||||
if (symbol.empty())
|
||||
return false;
|
||||
|
||||
/* [A-Za-z_] */
|
||||
unsigned char first = symbol[0];
|
||||
if (!((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z') || first == '_'))
|
||||
return false;
|
||||
|
||||
/* [A-Za-z0-9-_+]* */
|
||||
for (unsigned char c : symbol.substr(1)) {
|
||||
if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_'
|
||||
|| c == '+')
|
||||
continue;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void getDerivations(
|
||||
EvalState & state,
|
||||
@@ -378,6 +399,8 @@ static void getDerivations(
|
||||
Done & done,
|
||||
bool ignoreAssertionFailures)
|
||||
{
|
||||
auto _level = state.addCallDepth(vIn.determinePos(noPos));
|
||||
|
||||
Value v;
|
||||
state.autoCallFunction(autoArgs, vIn, v);
|
||||
|
||||
@@ -400,7 +423,7 @@ static void getDerivations(
|
||||
std::string_view symbol{state.symbols[i->name]};
|
||||
try {
|
||||
debug("evaluating attribute '%1%'", symbol);
|
||||
if (!std::regex_match(symbol.begin(), symbol.end(), attrRegex))
|
||||
if (!isAttrPathComponent(symbol))
|
||||
continue;
|
||||
std::string pathPrefix2 = addToPath(pathPrefix, symbol);
|
||||
if (combineChannels)
|
||||
|
||||
@@ -54,6 +54,20 @@ MakeError(TypeError, EvalError);
|
||||
MakeError(UndefinedVarError, EvalError);
|
||||
MakeError(MissingArgumentError, EvalError);
|
||||
MakeError(InfiniteRecursionError, EvalError);
|
||||
|
||||
/**
|
||||
* Resource exhaustion error when evaluation exceeds max-call-depth.
|
||||
* Inherits from EvalBaseError (not EvalError) because resource exhaustion
|
||||
* should not be cached.
|
||||
*/
|
||||
struct StackOverflowError : public EvalBaseError
|
||||
{
|
||||
StackOverflowError(EvalState & state)
|
||||
: EvalBaseError(state, "stack overflow; max-call-depth exceeded")
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
MakeError(IFDError, EvalBaseError);
|
||||
|
||||
struct InvalidPathError : public EvalError
|
||||
|
||||
@@ -139,7 +139,7 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
|
||||
inline CallDepth EvalState::addCallDepth(const PosIdx pos)
|
||||
{
|
||||
if (callDepth > settings.maxCallDepth)
|
||||
error<EvalBaseError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
|
||||
error<StackOverflowError>().atPos(pos).debugThrow();
|
||||
|
||||
return CallDepth(callDepth);
|
||||
};
|
||||
|
||||
@@ -1002,6 +1002,12 @@ public:
|
||||
[[nodiscard]] StringMap
|
||||
realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
|
||||
|
||||
/**
|
||||
* Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`.
|
||||
*/
|
||||
SourcePath realisePath(
|
||||
const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full);
|
||||
|
||||
/**
|
||||
* Realise the given string with context, and return the string with outputs instead of downstream output
|
||||
* placeholders.
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
class EvalState;
|
||||
|
||||
/**
|
||||
* Print a value in the deprecated format used by `nix-instantiate --eval` and
|
||||
* `nix-env` (for manifests).
|
||||
@@ -15,7 +17,6 @@ namespace nix {
|
||||
*
|
||||
* See: https://github.com/NixOS/nix/issues/9730
|
||||
*/
|
||||
void printAmbiguous(
|
||||
Value & v, const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen, int depth);
|
||||
void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set<const void *> * seen, size_t depth = 0);
|
||||
|
||||
} // namespace nix
|
||||
+40
-34
@@ -157,24 +157,20 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
|
||||
return res;
|
||||
}
|
||||
|
||||
static SourcePath realisePath(
|
||||
EvalState & state,
|
||||
const PosIdx pos,
|
||||
Value & v,
|
||||
std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full)
|
||||
SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks)
|
||||
{
|
||||
NixStringContext context;
|
||||
|
||||
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
|
||||
auto path = coerceToPath(noPos, v, context, "while realising the context of a path");
|
||||
|
||||
try {
|
||||
if (!context.empty() && path.accessor == state.rootFS) {
|
||||
auto rewrites = state.realiseContext(context);
|
||||
if (!context.empty() && path.accessor == rootFS) {
|
||||
auto rewrites = realiseContext(context);
|
||||
path = {path.accessor, CanonPath(rewriteStrings(path.path.abs(), rewrites))};
|
||||
}
|
||||
return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path;
|
||||
} catch (Error & e) {
|
||||
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
|
||||
e.addTrace(positions[pos], "while realising the context of path '%s'", path);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -294,7 +290,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
|
||||
argument. */
|
||||
static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
|
||||
{
|
||||
auto path = realisePath(state, pos, vPath, std::nullopt);
|
||||
auto path = state.realisePath(pos, vPath, std::nullopt);
|
||||
auto path2 = path.path.abs();
|
||||
|
||||
// FIXME
|
||||
@@ -447,7 +443,7 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
|
||||
/* Load a ValueInitializer from a DSO and return whatever it initializes */
|
||||
void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = realisePath(state, pos, *args[0]);
|
||||
auto path = state.realisePath(pos, *args[0]);
|
||||
|
||||
std::string sym(
|
||||
state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative"));
|
||||
@@ -1971,7 +1967,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value ** args,
|
||||
arg.type() == nString && (arg.string_view().ends_with("/") || arg.string_view().ends_with("/."));
|
||||
|
||||
auto symlinkResolution = mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors;
|
||||
auto path = realisePath(state, pos, arg, symlinkResolution);
|
||||
auto path = state.realisePath(pos, arg, symlinkResolution);
|
||||
|
||||
auto st = path.maybeLstat();
|
||||
auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
|
||||
@@ -2078,7 +2074,7 @@ static RegisterPrimOp primop_dirOf({
|
||||
/* Return the contents of a file as a string. */
|
||||
static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = realisePath(state, pos, *args[0]);
|
||||
auto path = state.realisePath(pos, *args[0]);
|
||||
auto s = path.readFile();
|
||||
if (s.find((char) 0) != std::string::npos)
|
||||
state.error<EvalError>("the contents of the file '%1%' cannot be represented as a Nix string", path)
|
||||
@@ -2313,7 +2309,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
|
||||
if (!ha)
|
||||
state.error<EvalError>("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow();
|
||||
|
||||
auto path = realisePath(state, pos, *args[1]);
|
||||
auto path = state.realisePath(pos, *args[1]);
|
||||
|
||||
v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
|
||||
}
|
||||
@@ -2365,7 +2361,7 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
|
||||
|
||||
static void prim_readFileType(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = realisePath(state, pos, *args[0], std::nullopt);
|
||||
auto path = state.realisePath(pos, *args[0], std::nullopt);
|
||||
/* Retrieve the directory entry type and stringize it. */
|
||||
v = fileTypeToString(state, path.lstat().type);
|
||||
}
|
||||
@@ -2383,7 +2379,7 @@ static RegisterPrimOp primop_readFileType({
|
||||
/* Read a directory (without . or ..) */
|
||||
static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = realisePath(state, pos, *args[0]);
|
||||
auto path = state.realisePath(pos, *args[0]);
|
||||
|
||||
// Retrieve directory entries for all nodes in a directory.
|
||||
// This is similar to `getFileType` but is optimized to reduce system calls
|
||||
@@ -3002,16 +2998,19 @@ static RegisterPrimOp primop_path({
|
||||
|
||||
- recursive\
|
||||
When `false`, when `path` is added to the store it is with a
|
||||
flat hash, rather than a hash of the NAR serialization of the
|
||||
file. Thus, `path` must refer to a regular file, not a
|
||||
[flat hash](@docroot@/store/file-system-object/content-address.md#serial-flat),
|
||||
rather than a hash of the
|
||||
[NAR serialization](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
|
||||
of the file. Thus, `path` must refer to a regular file, not a
|
||||
directory. This allows similar behavior to `fetchurl`. Defaults
|
||||
to `true`.
|
||||
|
||||
- sha256\
|
||||
When provided, this is the expected hash of the file at the
|
||||
path. Evaluation fails if the hash is incorrect, and
|
||||
providing a hash allows `builtins.path` to be used even when the
|
||||
`pure-eval` nix config option is on.
|
||||
When provided, this is the expected
|
||||
[content hash](@docroot@/store/file-system-object/content-address.md)
|
||||
of the path. Evaluation fails if the hash is incorrect,
|
||||
and providing a hash allows `builtins.path` to be used even
|
||||
when the `pure-eval` nix config option is on.
|
||||
)",
|
||||
.fun = prim_path,
|
||||
});
|
||||
@@ -4709,21 +4708,28 @@ static RegisterPrimOp primop_convertHash({
|
||||
|
||||
struct RegexCache
|
||||
{
|
||||
boost::concurrent_flat_map<std::string, std::regex, StringViewHash, std::equal_to<>> cache;
|
||||
|
||||
std::regex get(std::string_view re)
|
||||
struct Entry
|
||||
{
|
||||
std::regex regex;
|
||||
/* No std::regex constructor overload from std::string_view, but can be constructed
|
||||
from a pointer + size or an iterator range. */
|
||||
ref<const std::regex> regex;
|
||||
|
||||
Entry(const char * s, size_t count)
|
||||
: regex(make_ref<const std::regex>(s, count, std::regex::extended))
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
boost::concurrent_flat_map<std::string, Entry, StringViewHash, std::equal_to<>> cache;
|
||||
|
||||
ref<const std::regex> get(std::string_view re)
|
||||
{
|
||||
std::optional<ref<const std::regex>> regex;
|
||||
cache.try_emplace_and_cvisit(
|
||||
re,
|
||||
/*s=*/re.data(),
|
||||
/*count=*/re.size(),
|
||||
std::regex::extended,
|
||||
[®ex](const auto & kv) { regex = kv.second; },
|
||||
[®ex](const auto & kv) { regex = kv.second; });
|
||||
return regex;
|
||||
[®ex](const auto & kv) { regex = kv.second.regex; },
|
||||
[®ex](const auto & kv) { regex = kv.second.regex; });
|
||||
return *regex;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -4745,7 +4751,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
|
||||
|
||||
std::cmatch match;
|
||||
if (!std::regex_match(str.begin(), str.end(), match, regex)) {
|
||||
if (!std::regex_match(str.begin(), str.end(), match, *regex)) {
|
||||
v.mkNull();
|
||||
return;
|
||||
}
|
||||
@@ -4818,7 +4824,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
const auto str =
|
||||
state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
|
||||
|
||||
auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
|
||||
auto begin = std::cregex_iterator(str.begin(), str.end(), *regex);
|
||||
auto end = std::cregex_iterator();
|
||||
|
||||
// Any matches results are surrounded by non-matching results.
|
||||
|
||||
@@ -243,7 +243,7 @@ static RegisterPrimOp primop_fetchClosure({
|
||||
```nix
|
||||
builtins.fetchClosure {
|
||||
fromStore = "https://cache.nixos.org";
|
||||
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
|
||||
fromPath = /nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1;
|
||||
toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
|
||||
}
|
||||
```
|
||||
@@ -258,8 +258,8 @@ static RegisterPrimOp primop_fetchClosure({
|
||||
use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
|
||||
|
||||
```console
|
||||
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
|
||||
rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
|
||||
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1
|
||||
rewrote '/nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
|
||||
```
|
||||
|
||||
Alternatively, set `toPath = ""` and find the correct `toPath` in the error message.
|
||||
@@ -271,7 +271,7 @@ static RegisterPrimOp primop_fetchClosure({
|
||||
```nix
|
||||
builtins.fetchClosure {
|
||||
fromStore = "https://cache.nixos.org";
|
||||
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
|
||||
fromPath = /nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1;
|
||||
inputAddressed = true;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -2,19 +2,17 @@
|
||||
#include "nix/expr/print.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/expr/eval.hh"
|
||||
#include "nix/expr/eval-error.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
// See: https://github.com/NixOS/nix/issues/9730
|
||||
void printAmbiguous(
|
||||
Value & v, const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen, int depth)
|
||||
void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set<const void *> * seen, size_t depth)
|
||||
{
|
||||
checkInterrupt();
|
||||
|
||||
if (depth <= 0) {
|
||||
str << "«too deep»";
|
||||
return;
|
||||
}
|
||||
if (depth > state.settings.maxCallDepth)
|
||||
state.error<StackOverflowError>().atPos(v.determinePos(noPos)).debugThrow();
|
||||
switch (v.type()) {
|
||||
case nInt:
|
||||
str << v.integer();
|
||||
@@ -36,9 +34,9 @@ void printAmbiguous(
|
||||
str << "«repeated»";
|
||||
else {
|
||||
str << "{ ";
|
||||
for (auto & i : v.attrs()->lexicographicOrder(symbols)) {
|
||||
str << symbols[i->name] << " = ";
|
||||
printAmbiguous(*i->value, symbols, str, seen, depth - 1);
|
||||
for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
|
||||
str << state.symbols[i->name] << " = ";
|
||||
printAmbiguous(state, *i->value, str, seen, depth + 1);
|
||||
str << "; ";
|
||||
}
|
||||
str << "}";
|
||||
@@ -54,7 +52,7 @@ void printAmbiguous(
|
||||
str << "[ ";
|
||||
for (auto v2 : v.listView()) {
|
||||
if (v2)
|
||||
printAmbiguous(*v2, symbols, str, seen, depth - 1);
|
||||
printAmbiguous(state, *v2, str, seen, depth + 1);
|
||||
else
|
||||
str << "(nullptr)";
|
||||
str << " ";
|
||||
|
||||
@@ -537,6 +537,16 @@ private:
|
||||
output.flush();
|
||||
checkInterrupt();
|
||||
|
||||
// Catch infinite recursion before it overflows the C++ stack.
|
||||
// Non-cyclic structures can be infinitely deep when values are
|
||||
// lazily produced (e.g., `let f = n: { inner = f (n + 1); }; in f 0`).
|
||||
// We check print depth against max-call-depth rather than incrementing
|
||||
// the callDepth counter, because accessing an attribute is not a call.
|
||||
// Other places do increment callDepth for simplicity, but that is
|
||||
// technically incorrect.
|
||||
if (depth > state.settings.maxCallDepth)
|
||||
state.error<StackOverflowError>().atPos(v.determinePos(noPos)).debugThrow();
|
||||
|
||||
try {
|
||||
if (options.force) {
|
||||
state.forceValue(v, v.determinePos(noPos));
|
||||
@@ -592,6 +602,11 @@ private:
|
||||
printUnknown();
|
||||
break;
|
||||
}
|
||||
} catch (StackOverflowError &) {
|
||||
// Always re-throw because stack overflow is a serious condition
|
||||
// that expressions should avoid, unlike say `throw`, which can
|
||||
// be part of legitimate expression patterns.
|
||||
throw;
|
||||
} catch (Error & e) {
|
||||
if (options.errors == ErrorPrintBehavior::Throw
|
||||
|| (options.errors == ErrorPrintBehavior::ThrowTopLevel && depth == 0)) {
|
||||
|
||||
@@ -66,6 +66,8 @@ static void printValueAsXML(
|
||||
{
|
||||
checkInterrupt();
|
||||
|
||||
auto _level = state.addCallDepth(pos);
|
||||
|
||||
if (strict)
|
||||
state.forceValue(v, pos);
|
||||
|
||||
|
||||
@@ -115,9 +115,10 @@ TEST_F(GitUtilsTest, sink_hardlink)
|
||||
|
||||
try {
|
||||
sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello"));
|
||||
sink->flush();
|
||||
FAIL() << "Expected an exception";
|
||||
} catch (const nix::Error & e) {
|
||||
ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target"));
|
||||
ASSERT_THAT(e.msg(), testing::HasSubstr("does not exist"));
|
||||
ASSERT_THAT(e.msg(), testing::HasSubstr("/hello"));
|
||||
ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link"));
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ struct CacheImpl : Cache
|
||||
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
|
||||
createDirs(dirOf(dbPath));
|
||||
|
||||
state->db = SQLite(dbPath);
|
||||
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
|
||||
@@ -489,11 +489,11 @@ void InputScheme::clone(
|
||||
const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
|
||||
{
|
||||
if (std::filesystem::exists(destDir))
|
||||
throw Error("cannot clone into existing path %s", destDir);
|
||||
throw Error("cannot clone into existing path %s", PathFmt(destDir));
|
||||
|
||||
auto [accessor, input2] = getAccessor(settings, store, input);
|
||||
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), PathFmt(destDir)));
|
||||
|
||||
RestoreSink sink(/*startFsync=*/false);
|
||||
sink.dstPath = destDir;
|
||||
|
||||
@@ -10,12 +10,6 @@ std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(co
|
||||
return next->getPhysicalPath(prefix / path);
|
||||
}
|
||||
|
||||
std::string FilteringSourceAccessor::readFile(const CanonPath & path)
|
||||
{
|
||||
checkAccess(path);
|
||||
return next->readFile(prefix / path);
|
||||
}
|
||||
|
||||
void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||
{
|
||||
checkAccess(path);
|
||||
|
||||
@@ -273,7 +273,7 @@ void Fetch::fetch(
|
||||
+ "/" + pointer->oid;
|
||||
std::filesystem::path cachePath = cacheDir / key;
|
||||
if (pathExists(cachePath)) {
|
||||
debug("using cache entry %s -> %s", key, cachePath);
|
||||
debug("using cache entry %s -> %s", key, PathFmt(cachePath));
|
||||
sink(readFile(cachePath));
|
||||
return;
|
||||
}
|
||||
@@ -301,7 +301,7 @@ void Fetch::fetch(
|
||||
sizeCallback(size);
|
||||
downloadToSink(ourl, authHeader, sink, sha256, size);
|
||||
|
||||
debug("creating cache entry %s -> %s", key, cachePath);
|
||||
debug("creating cache entry %s -> %s", key, PathFmt(cachePath));
|
||||
if (!pathExists(cachePath.parent_path()))
|
||||
createDirs(cachePath.parent_path());
|
||||
writeFile(cachePath, sink.s);
|
||||
|
||||
+249
-223
@@ -209,14 +209,14 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
|
||||
return;
|
||||
|
||||
if (!options.create)
|
||||
throw Error("Git repository %s does not exist.", path);
|
||||
throw Error("Git repository %s does not exist.", PathFmt(path));
|
||||
|
||||
std::filesystem::path tmpDir = createTempDir(path.parent_path());
|
||||
AutoDelete delTmpDir(tmpDir, true);
|
||||
Repository tmpRepo;
|
||||
|
||||
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
|
||||
throw Error("creating Git repository %s: %s", path, git_error_last()->message);
|
||||
throw Error("creating Git repository %s: %s", PathFmt(path), git_error_last()->message);
|
||||
try {
|
||||
std::filesystem::rename(tmpDir, path);
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
@@ -226,7 +226,7 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
|
||||
|| e.code() == std::errc::directory_not_empty) {
|
||||
return;
|
||||
} else
|
||||
throw SysError("moving temporary git repository from %s to %s", tmpDir, path);
|
||||
throw SysError("moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path));
|
||||
}
|
||||
// we successfully moved the repository, so the temporary directory no longer exists.
|
||||
delTmpDir.cancel();
|
||||
@@ -266,7 +266,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
|
||||
initRepoAtomically(path, options);
|
||||
if (git_repository_open(Setter(repo), path.string().c_str()))
|
||||
throw Error("opening Git repository %s: %s", path, git_error_last()->message);
|
||||
throw Error("opening Git repository %s: %s", PathFmt(path), git_error_last()->message);
|
||||
|
||||
ObjectDb odb;
|
||||
if (options.packfilesOnly) {
|
||||
@@ -369,7 +369,26 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
{
|
||||
// TODO: as an optimization, it would be nice to include `this` in the pool.
|
||||
return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
|
||||
return make_ref<GitRepoImpl>(path, options);
|
||||
auto repo = make_ref<GitRepoImpl>(path, options);
|
||||
|
||||
/* Monkey-patching the pack backend to only read the pack directory
|
||||
once. Otherwise it will do a readdir for each added oid when it's
|
||||
not found and that translates to ~6 syscalls. Since we are never
|
||||
writing pack files until flushing we can force the odb backend to
|
||||
read the directory just once. It's very convenient that the vtable is
|
||||
semi-public interface and is up for grabs.
|
||||
|
||||
This is purely an optimization for our use-case with a tarball cache.
|
||||
libgit2 calls refresh() if the backend provides it when an oid isn't found.
|
||||
We are only writing objects to a mempack (it has higher priority) and there isn't
|
||||
a realistic use-case where a previously missing object would appear from thin air
|
||||
on the disk (unless another process happens to be unpacking a similar tarball to
|
||||
the cache at the same time, but that's a very unrealistic scenario).
|
||||
*/
|
||||
if (auto * backend = repo->packBackend)
|
||||
backend->refresh = nullptr;
|
||||
|
||||
return repo;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -668,6 +687,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
keyDecoded = base64::decode(k.key);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while decoding public key '%s' used for git signature", k.key);
|
||||
throw;
|
||||
}
|
||||
auto fingerprint =
|
||||
trim(hashString(HashAlgorithm::SHA256, keyDecoded).to_string(nix::HashFormat::Base64, false), "=");
|
||||
@@ -746,7 +766,7 @@ struct GitSourceAccessor : SourceAccessor
|
||||
{
|
||||
}
|
||||
|
||||
std::string readBlob(const CanonPath & path, bool symlink)
|
||||
void readBlob(const CanonPath & path, bool symlink, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||
{
|
||||
auto state(state_.lock());
|
||||
|
||||
@@ -765,16 +785,22 @@ struct GitSourceAccessor : SourceAccessor
|
||||
e.addTrace({}, "while smudging git-lfs file '%s'", path);
|
||||
throw;
|
||||
}
|
||||
return s.s;
|
||||
sizeCallback(s.s.size());
|
||||
StringSource source{s.s};
|
||||
source.drainInto(sink);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
|
||||
auto view = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
|
||||
sizeCallback(view.size());
|
||||
StringSource source{view};
|
||||
source.drainInto(sink);
|
||||
}
|
||||
|
||||
std::string readFile(const CanonPath & path) override
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
|
||||
{
|
||||
return readBlob(path, false);
|
||||
return readBlob(path, false, sink, sizeCallback);
|
||||
}
|
||||
|
||||
bool pathExists(const CanonPath & path) override
|
||||
@@ -841,7 +867,9 @@ struct GitSourceAccessor : SourceAccessor
|
||||
|
||||
std::string readLink(const CanonPath & path) override
|
||||
{
|
||||
return readBlob(path, true);
|
||||
StringSink s;
|
||||
readBlob(path, true, s, [&](uint64_t size) { s.s.reserve(size); });
|
||||
return std::move(s.s);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1058,185 +1086,155 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
{
|
||||
ref<GitRepoImpl> repo;
|
||||
|
||||
struct PendingDir
|
||||
{
|
||||
std::string name;
|
||||
TreeBuilder builder;
|
||||
};
|
||||
Pool<GitRepoImpl> repoPool;
|
||||
|
||||
std::vector<PendingDir> pendingDirs;
|
||||
unsigned int concurrency = std::min(std::thread::hardware_concurrency(), 10U);
|
||||
|
||||
/**
|
||||
* Temporary buffer used by createRegularFile for storing small file contents.
|
||||
*/
|
||||
std::string regularFileContentsBuffer;
|
||||
ThreadPool workers{concurrency};
|
||||
|
||||
/**
|
||||
* If repo has a non-null packBackend, this has a copy of the refresh function
|
||||
* from the backend virtual table. This is needed to restore it after we've flushed
|
||||
* the sink. We modify it to avoid unnecessary I/O on non-existent oids.
|
||||
*/
|
||||
decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
|
||||
/** Total file contents in flight. */
|
||||
std::atomic<size_t> totalBufSize{0};
|
||||
|
||||
void pushBuilder(std::string name)
|
||||
{
|
||||
const git_tree_entry * entry;
|
||||
Tree prevTree = nullptr;
|
||||
|
||||
if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
|
||||
/* Clone a tree that we've already finished. This happens
|
||||
if a tarball has directory entries that are not
|
||||
contiguous. */
|
||||
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
|
||||
throw Error("parent of '%s' is not a directory", name);
|
||||
|
||||
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
|
||||
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
|
||||
}
|
||||
|
||||
git_treebuilder * b;
|
||||
if (git_treebuilder_new(&b, *repo, prevTree.get()))
|
||||
throw Error("creating a tree builder: %s", git_error_last()->message);
|
||||
pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
|
||||
};
|
||||
static constexpr std::size_t maxBufSize = 16 * 1024 * 1024;
|
||||
|
||||
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
|
||||
: repo(repo)
|
||||
, repoPool(repo->getPool())
|
||||
{
|
||||
/* Monkey-patching the pack backend to only read the pack directory
|
||||
once. Otherwise it will do a readdir for each added oid when it's
|
||||
not found and that translates to ~6 syscalls. Since we are never
|
||||
writing pack files until flushing we can force the odb backend to
|
||||
read the directory just once. It's very convenient that the vtable is
|
||||
semi-public interface and is up for grabs.
|
||||
|
||||
This is purely an optimization for our use-case with a tarball cache.
|
||||
libgit2 calls refresh() if the backend provides it when an oid isn't found.
|
||||
We are only writing objects to a mempack (it has higher priority) and there isn't
|
||||
a realistic use-case where a previously missing object would appear from thin air
|
||||
on the disk (unless another process happens to be unpacking a similar tarball to
|
||||
the cache at the same time, but that's a very unrealistic scenario).
|
||||
*/
|
||||
if (auto * backend = repo->packBackend) {
|
||||
if (backend->refresh(backend)) /* Refresh just once manually. */
|
||||
throw Error("refreshing packfiles: %s", git_error_last()->message);
|
||||
/* Save the function pointer to restore it later in flush() and
|
||||
unset it in the vtable. libgit2 does nothing if it's a nullptr:
|
||||
https://github.com/libgit2/libgit2/blob/58d9363f02f1fa39e46d49b604f27008e75b72f2/src/libgit2/odb.c#L1922
|
||||
*/
|
||||
packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
|
||||
}
|
||||
pushBuilder("");
|
||||
}
|
||||
|
||||
std::pair<git_oid, std::string> popBuilder()
|
||||
~GitFileSystemObjectSinkImpl()
|
||||
{
|
||||
assert(!pendingDirs.empty());
|
||||
auto pending = std::move(pendingDirs.back());
|
||||
git_oid oid;
|
||||
if (git_treebuilder_write(&oid, pending.builder.get()))
|
||||
throw Error("creating a tree object: %s", git_error_last()->message);
|
||||
pendingDirs.pop_back();
|
||||
return {oid, pending.name};
|
||||
// Make sure the worker threads are destroyed before any state
|
||||
// they're referring to.
|
||||
workers.shutdown();
|
||||
}
|
||||
|
||||
struct Child;
|
||||
|
||||
/// A directory to be written as a Git tree.
|
||||
struct Directory
|
||||
{
|
||||
std::map<std::string, Child> children;
|
||||
std::optional<git_oid> oid;
|
||||
|
||||
Child & lookup(const CanonPath & path)
|
||||
{
|
||||
assert(!path.isRoot());
|
||||
auto parent = path.parent();
|
||||
auto cur = this;
|
||||
for (auto & name : *parent) {
|
||||
auto i = cur->children.find(std::string(name));
|
||||
if (i == cur->children.end())
|
||||
throw Error("path '%s' does not exist", path);
|
||||
auto dir = std::get_if<Directory>(&i->second.file);
|
||||
if (!dir)
|
||||
throw Error("path '%s' has a non-directory parent", path);
|
||||
cur = dir;
|
||||
}
|
||||
|
||||
auto i = cur->children.find(std::string(*path.baseName()));
|
||||
if (i == cur->children.end())
|
||||
throw Error("path '%s' does not exist", path);
|
||||
return i->second;
|
||||
}
|
||||
};
|
||||
|
||||
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
|
||||
size_t nextId = 0; // for Child.id
|
||||
|
||||
struct Child
|
||||
{
|
||||
assert(!pendingDirs.empty());
|
||||
auto & pending = pendingDirs.back();
|
||||
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
|
||||
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
|
||||
git_filemode_t mode;
|
||||
std::variant<Directory, git_oid> file;
|
||||
|
||||
/// Sequential numbering of the file in the tarball. This is
|
||||
/// used to make sure we only import the latest version of a
|
||||
/// path.
|
||||
size_t id{0};
|
||||
};
|
||||
|
||||
void updateBuilders(std::span<const std::string> names)
|
||||
struct State
|
||||
{
|
||||
// Find the common prefix of pendingDirs and names.
|
||||
size_t prefixLen = 0;
|
||||
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
|
||||
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
|
||||
break;
|
||||
Directory root;
|
||||
};
|
||||
|
||||
// Finish the builders that are not part of the common prefix.
|
||||
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
|
||||
auto [oid, name] = popBuilder();
|
||||
addToTree(name, oid, GIT_FILEMODE_TREE);
|
||||
Sync<State> _state;
|
||||
|
||||
void addNode(State & state, const CanonPath & path, Child && child)
|
||||
{
|
||||
assert(!path.isRoot());
|
||||
auto parent = path.parent();
|
||||
|
||||
Directory * cur = &state.root;
|
||||
|
||||
for (auto & i : *parent) {
|
||||
auto child = std::get_if<Directory>(
|
||||
&cur->children.emplace(std::string(i), Child{GIT_FILEMODE_TREE, {Directory()}}).first->second.file);
|
||||
assert(child);
|
||||
cur = child;
|
||||
}
|
||||
|
||||
// Create builders for the new directories.
|
||||
for (auto n = prefixLen; n < names.size(); ++n)
|
||||
pushBuilder(names[n]);
|
||||
};
|
||||
std::string name(*path.baseName());
|
||||
|
||||
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
|
||||
{
|
||||
std::span<const std::string> pathComponents2{pathComponents};
|
||||
|
||||
updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
|
||||
|
||||
return true;
|
||||
if (auto prev = cur->children.find(name); prev == cur->children.end() || prev->second.id < child.id)
|
||||
cur->children.insert_or_assign(name, std::move(child));
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
checkInterrupt();
|
||||
|
||||
/* Multithreaded blob writing. We read the incoming file data into memory and asynchronously write it to a Git
|
||||
blob object. However, to avoid unbounded memory usage, if the amount of data in flight exceeds a threshold,
|
||||
we switch to writing directly to a Git write stream. */
|
||||
|
||||
using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
|
||||
if (stream)
|
||||
stream->free(stream);
|
||||
})>;
|
||||
|
||||
/* Maximum file size that gets buffered in memory before flushing to a WriteStream,
|
||||
that's backed by a temporary objects/streamed_git2_* file. We should avoid that
|
||||
for common cases, since creating (and deleting) a temporary file for each blob
|
||||
is insanely expensive. */
|
||||
static constexpr std::size_t maxBufferSize = 1024 * 1024; /* 1 MiB */
|
||||
|
||||
struct CRF : CreateRegularFileSink
|
||||
{
|
||||
const CanonPath & path;
|
||||
GitFileSystemObjectSinkImpl & back;
|
||||
CanonPath path;
|
||||
GitFileSystemObjectSinkImpl & parent;
|
||||
WriteStream stream;
|
||||
std::string & contents;
|
||||
std::optional<decltype(parent.repoPool)::Handle> repo;
|
||||
|
||||
std::string contents;
|
||||
bool executable = false;
|
||||
|
||||
CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, std::string & regularFileContentsBuffer)
|
||||
: path(path)
|
||||
, back(back)
|
||||
, stream(nullptr)
|
||||
, contents(regularFileContentsBuffer)
|
||||
CRF(CanonPath path, GitFileSystemObjectSinkImpl & parent)
|
||||
: path(std::move(path))
|
||||
, parent(parent)
|
||||
{
|
||||
contents.clear();
|
||||
}
|
||||
|
||||
void writeToStream(std::string_view data)
|
||||
~CRF()
|
||||
{
|
||||
/* Lazily create the stream. */
|
||||
if (!stream) {
|
||||
::git_writestream * stream2 = nullptr;
|
||||
if (git_blob_create_from_stream(&stream2, *back.repo, nullptr))
|
||||
throw Error("creating a blob stream object: %s", git_error_last()->message);
|
||||
stream = WriteStream{stream2};
|
||||
assert(stream);
|
||||
}
|
||||
|
||||
if (stream->write(stream.get(), data.data(), data.size()))
|
||||
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
|
||||
parent.totalBufSize -= contents.size();
|
||||
}
|
||||
|
||||
void operator()(std::string_view data) override
|
||||
{
|
||||
/* Already in slow path. Just write to the slow stream. */
|
||||
if (stream) {
|
||||
writeToStream(data);
|
||||
return;
|
||||
}
|
||||
if (!stream) {
|
||||
contents.append(data);
|
||||
parent.totalBufSize += data.size();
|
||||
|
||||
contents += data;
|
||||
if (contents.size() > maxBufferSize) {
|
||||
writeToStream(contents); /* Will initialize stream. */
|
||||
contents.clear();
|
||||
if (parent.totalBufSize > parent.maxBufSize) {
|
||||
repo.emplace(parent.repoPool.get());
|
||||
|
||||
if (git_blob_create_from_stream(Setter(stream), **repo, nullptr))
|
||||
throw Error("creating a blob stream object: %s", git_error_last()->message);
|
||||
|
||||
if (stream->write(stream.get(), contents.data(), contents.size()))
|
||||
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
|
||||
|
||||
parent.totalBufSize -= contents.size();
|
||||
contents.clear();
|
||||
}
|
||||
} else {
|
||||
if (stream->write(stream.get(), data.data(), data.size()))
|
||||
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1244,112 +1242,140 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
{
|
||||
executable = true;
|
||||
}
|
||||
} crf{path, *this, regularFileContentsBuffer};
|
||||
};
|
||||
|
||||
func(crf);
|
||||
auto crf = std::make_shared<CRF>(path, *this);
|
||||
|
||||
git_oid oid;
|
||||
if (crf.stream) {
|
||||
/* Call .release(), since git_blob_create_from_stream_commit
|
||||
func(*crf);
|
||||
|
||||
auto id = nextId++;
|
||||
|
||||
if (crf->stream) {
|
||||
/* Finish the slow path by creating the blob object synchronously.
|
||||
Call .release(), since git_blob_create_from_stream_commit
|
||||
acquires ownership and frees the stream. */
|
||||
if (git_blob_create_from_stream_commit(&oid, crf.stream.release()))
|
||||
git_oid oid;
|
||||
if (git_blob_create_from_stream_commit(&oid, crf->stream.release()))
|
||||
throw Error("creating a blob object for '%s': %s", path, git_error_last()->message);
|
||||
} else {
|
||||
if (git_blob_create_from_buffer(&oid, *repo, crf.contents.data(), crf.contents.size()))
|
||||
throw Error(
|
||||
"creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
|
||||
addNode(
|
||||
*_state.lock(),
|
||||
crf->path,
|
||||
Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id});
|
||||
return;
|
||||
}
|
||||
|
||||
addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
|
||||
/* Fast path: create the blob object in a separate thread. */
|
||||
workers.enqueue([this, crf{std::move(crf)}, id]() {
|
||||
auto repo(repoPool.get());
|
||||
|
||||
git_oid oid;
|
||||
if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size()))
|
||||
throw Error(
|
||||
"creating a blob object for '%s' from in-memory buffer: %s", crf->path, git_error_last()->message);
|
||||
|
||||
addNode(
|
||||
*_state.lock(),
|
||||
crf->path,
|
||||
Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id});
|
||||
});
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
(void) prepareDirs(pathComponents, true);
|
||||
if (path.isRoot())
|
||||
return;
|
||||
auto state(_state.lock());
|
||||
addNode(*state, path, {GIT_FILEMODE_TREE, Directory()});
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
workers.enqueue([this, path, target]() {
|
||||
auto repo(repoPool.get());
|
||||
|
||||
git_oid oid;
|
||||
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
|
||||
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
|
||||
git_oid oid;
|
||||
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
|
||||
throw Error(
|
||||
"creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
|
||||
|
||||
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
|
||||
auto state(_state.lock());
|
||||
addNode(*state, path, Child{GIT_FILEMODE_LINK, oid});
|
||||
});
|
||||
}
|
||||
|
||||
std::map<CanonPath, CanonPath> hardLinks;
|
||||
|
||||
void createHardlink(const CanonPath & path, const CanonPath & target) override
|
||||
{
|
||||
std::vector<std::string> pathComponents;
|
||||
for (auto & c : path)
|
||||
pathComponents.emplace_back(c);
|
||||
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
// We can't just look up the path from the start of the root, since
|
||||
// some parent directories may not have finished yet, so we compute
|
||||
// a relative path that helps us find the right git_tree_builder or object.
|
||||
auto relTarget = CanonPath(path).parent()->makeRelative(target);
|
||||
|
||||
auto dir = pendingDirs.rbegin();
|
||||
|
||||
// For each ../ component at the start, go up one directory.
|
||||
// CanonPath::makeRelative() always puts all .. elements at the start,
|
||||
// so they're all handled by this loop:
|
||||
std::string_view relTargetLeft(relTarget);
|
||||
while (hasPrefix(relTargetLeft, "../")) {
|
||||
if (dir == pendingDirs.rend())
|
||||
throw Error("invalid hard link target '%s' for path '%s'", target, path);
|
||||
++dir;
|
||||
relTargetLeft = relTargetLeft.substr(3);
|
||||
}
|
||||
if (dir == pendingDirs.rend())
|
||||
throw Error("invalid hard link target '%s' for path '%s'", target, path);
|
||||
|
||||
// Look up the remainder of the target, starting at the
|
||||
// top-most `git_treebuilder`.
|
||||
std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
|
||||
Object tree; // needed to keep `entry` alive
|
||||
const git_tree_entry * entry = nullptr;
|
||||
|
||||
for (auto & c : CanonPath(relTargetLeft)) {
|
||||
if (auto builder = std::get_if<git_treebuilder *>(&curDir)) {
|
||||
assert(*builder);
|
||||
if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str())))
|
||||
throw Error("cannot find hard link target '%s' for path '%s'", target, path);
|
||||
curDir = *git_tree_entry_id(entry);
|
||||
} else if (auto oid = std::get_if<git_oid>(&curDir)) {
|
||||
tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE);
|
||||
if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str())))
|
||||
throw Error("cannot find hard link target '%s' for path '%s'", target, path);
|
||||
curDir = *git_tree_entry_id(entry);
|
||||
}
|
||||
}
|
||||
|
||||
assert(entry);
|
||||
|
||||
addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
|
||||
hardLinks.insert_or_assign(path, target);
|
||||
}
|
||||
|
||||
Hash flush() override
|
||||
{
|
||||
updateBuilders({});
|
||||
workers.process();
|
||||
|
||||
auto [oid, _name] = popBuilder();
|
||||
|
||||
if (auto * backend = repo->packBackend) {
|
||||
/* We are done writing blobs, can restore refresh functionality. */
|
||||
backend->refresh = packfileOdbRefresh;
|
||||
/* Create hard links. */
|
||||
{
|
||||
auto state(_state.lock());
|
||||
for (auto & [path, target] : hardLinks) {
|
||||
if (target.isRoot())
|
||||
continue;
|
||||
try {
|
||||
auto child = state->root.lookup(target);
|
||||
auto oid = std::get_if<git_oid>(&child.file);
|
||||
if (!oid)
|
||||
throw Error("cannot create a hard link to a directory");
|
||||
addNode(*state, path, {child.mode, *oid});
|
||||
} catch (Error & e) {
|
||||
e.addTrace(nullptr, "while creating a hard link from '%s' to '%s'", path, target);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush all repo objects to disk.
|
||||
{
|
||||
auto repos = repoPool.clear();
|
||||
ThreadPool workers{repos.size()};
|
||||
for (auto & repo : repos)
|
||||
workers.enqueue([repo]() { repo->flush(); });
|
||||
workers.process();
|
||||
}
|
||||
|
||||
// Write the Git trees to disk. Would be nice to have this multithreaded too, but that's hard because a tree
|
||||
// can't refer to an object that hasn't been written yet. Also it doesn't make a big difference for performance.
|
||||
auto repo(repoPool.get());
|
||||
|
||||
[&](this const auto & visit, Directory & node) -> void {
|
||||
checkInterrupt();
|
||||
|
||||
// Write the child directories.
|
||||
for (auto & child : node.children)
|
||||
if (auto dir = std::get_if<Directory>(&child.second.file))
|
||||
visit(*dir);
|
||||
|
||||
// Write this directory.
|
||||
git_treebuilder * b;
|
||||
if (git_treebuilder_new(&b, *repo, nullptr))
|
||||
throw Error("creating a tree builder: %s", git_error_last()->message);
|
||||
TreeBuilder builder(b);
|
||||
|
||||
for (auto & [name, child] : node.children) {
|
||||
auto oid_p = std::get_if<git_oid>(&child.file);
|
||||
auto oid = oid_p ? *oid_p : std::get<Directory>(child.file).oid.value();
|
||||
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode))
|
||||
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
|
||||
}
|
||||
|
||||
git_oid oid;
|
||||
if (git_treebuilder_write(&oid, builder.get()))
|
||||
throw Error("creating a tree object: %s", git_error_last()->message);
|
||||
node.oid = oid;
|
||||
}(_state.lock()->root);
|
||||
|
||||
repo->flush();
|
||||
|
||||
return toHash(oid);
|
||||
return toHash(_state.lock()->root.oid.value());
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1407,7 +1433,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
|
||||
auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
|
||||
try {
|
||||
writeFull(fdTemp.get(), configS);
|
||||
} catch (SysError & e) {
|
||||
} catch (SystemError & e) {
|
||||
e.addTrace({}, "while writing .gitmodules file to temporary file");
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -72,10 +72,10 @@ std::optional<std::string> readHead(const std::filesystem::path & path)
|
||||
if (const auto parseResult = git::parseLsRemoteLine(line); parseResult && parseResult->reference == "HEAD") {
|
||||
switch (parseResult->kind) {
|
||||
case git::LsRemoteRefLine::Kind::Symbolic:
|
||||
debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path);
|
||||
debug("resolved HEAD ref '%s' for repo %s", parseResult->target, PathFmt(path));
|
||||
break;
|
||||
case git::LsRemoteRefLine::Kind::Object:
|
||||
debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path);
|
||||
debug("resolved HEAD rev '%s' for repo %s", parseResult->target, PathFmt(path));
|
||||
break;
|
||||
}
|
||||
return parseResult->target;
|
||||
@@ -753,9 +753,10 @@ struct GitInputScheme : InputScheme
|
||||
"\n"
|
||||
"git -C %2% add \"%1%\"",
|
||||
path.rel(),
|
||||
repoPath);
|
||||
PathFmt(repoPath));
|
||||
else
|
||||
return RestrictedPathError("Path '%s' does not exist in Git repository %s.", path.rel(), repoPath);
|
||||
return RestrictedPathError(
|
||||
"Path '%s' does not exist in Git repository %s.", path.rel(), PathFmt(repoPath));
|
||||
};
|
||||
}
|
||||
|
||||
@@ -848,7 +849,7 @@ struct GitInputScheme : InputScheme
|
||||
if (!input.getRev())
|
||||
setWriteTime(localRefFile, now, now);
|
||||
} catch (Error & e) {
|
||||
warn("could not update mtime for file %s: %s", localRefFile, e.info().msg);
|
||||
warn("could not update mtime for file %s: %s", PathFmt(localRefFile), e.info().msg);
|
||||
}
|
||||
if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref))
|
||||
warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
|
||||
|
||||
@@ -34,7 +34,7 @@ struct FilteringSourceAccessor : SourceAccessor
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
|
||||
|
||||
std::string readFile(const CanonPath & path) override;
|
||||
using SourceAccessor::readFile;
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
|
||||
|
||||
|
||||
@@ -154,7 +154,7 @@ struct PathInputScheme : InputScheme
|
||||
|
||||
time_t mtime = 0;
|
||||
if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", PathFmt(absPath)));
|
||||
// FIXME: try to substitute storePath.
|
||||
auto src = sinkToSource(
|
||||
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
|
||||
@@ -165,7 +165,6 @@ struct PathInputScheme : InputScheme
|
||||
|
||||
// To prevent `fetchToStore()` copying the path again to Nix
|
||||
// store, pre-create an entry in the fetcher cache.
|
||||
auto info = store.queryPathInfo(*storePath);
|
||||
accessor->fingerprint =
|
||||
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
|
||||
settings.getCache()->upsert(
|
||||
|
||||
@@ -114,15 +114,22 @@ static DownloadTarballResult downloadTarball_(
|
||||
// it is not in fact a tarball.
|
||||
if (url.scheme == "file") {
|
||||
std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path);
|
||||
if (!localPath.is_absolute()) {
|
||||
throw Error(
|
||||
"tarball '%s' must use an absolute path. "
|
||||
"The 'file' scheme does not support relative paths.",
|
||||
url);
|
||||
}
|
||||
if (!exists(localPath)) {
|
||||
throw Error("tarball '%s' does not exist.", localPath);
|
||||
throw Error("tarball %s does not exist.", PathFmt(localPath));
|
||||
}
|
||||
if (is_directory(localPath)) {
|
||||
if (exists(localPath / ".git")) {
|
||||
throw Error(
|
||||
"tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath);
|
||||
"tarball %s is a git repository, not a tarball. Please use `git+file` as the scheme.",
|
||||
PathFmt(localPath));
|
||||
}
|
||||
throw Error("tarball '%s' is a directory, not a file.", localPath);
|
||||
throw Error("tarball %s is a directory, not a file.", PathFmt(localPath));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loaded 100 of 375 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user