Compare commits

..
Author SHA1 Message Date
Sergei Zimmerman b1c0f416cb libutil: Make computeClosure async
This makes computeFSClosure much faster in practice, because the graph is now traversed in an async BFS manner, while
previously it serialised on each single path info query. For something like

rm ~/.cache/nix/binary-cache-v7.sqlite && nix path-info --store https://cache.nixos.org --recursive /nix/store/7zz3zmv2a0ssmgqlfhy4rsb6ii6z475a-stdenv-linux.drv

The difference is huge:

(Before)

Command being timed: "nix path-info --store https://cache.nixos.org --recursive /nix/store/7zz3zmv2a0ssmgqlfhy4rsb6ii6z475a-stdenv-linux.drv"
        User time (seconds): 0.07
        System time (seconds): 0.06
        Percent of CPU this job got: 1%
        Elapsed (wall clock) time (h:mm:ss or m:ss): 0:08.88

(After)

Command being timed: "build/src/nix/nix path-info --store https://cache.nixos.org --recursive /nix/store/7zz3zmv2a0ssmgqlfhy4rsb6ii6z475a-stdenv-linux.drv"
        User time (seconds): 0.07
        System time (seconds): 0.04
        Percent of CPU this job got: 22%
        Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.49

Basically 20x speedup with just processing stuff in an async manner.

Note that for now we have rather ad-hoc event loop just for
computeClosure. This seems perfectly fine for now, but in the future
we could extended it and possibly have a global even loop with multiple
threads handling it.

For now we have basic rate limiting <= 64 coroutines in flight which is
smaller than CURLMOPT_MAX_CONCURRENT_STREAMS (100 by default).
2025-12-19 07:28:44 +03:00
221 changed files with 1678 additions and 3636 deletions

No files matched your search

+81 -15
View File
@@ -164,7 +164,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- name: Download installer tarball
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
uses: actions/download-artifact@v7
with:
name: installer-${{matrix.os}}
path: out
@@ -197,6 +197,80 @@ jobs:
- run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
- run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
# Steps to test CI automation in your own fork.
# 1. Sign-up for https://hub.docker.com/
# 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
# 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
check_secrets:
permissions:
contents: none
name: Check presence of secrets
runs-on: ubuntu-24.04
outputs:
docker: ${{ steps.secret.outputs.docker }}
steps:
- name: Check for DockerHub secrets
id: secret
env:
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
run: |
echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
docker_push_image:
needs: [tests, check_secrets]
permissions:
contents: read
packages: write
if: >-
needs.check_secrets.outputs.docker == 'true' &&
github.event_name == 'push' &&
github.ref_name == 'master'
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: ./.github/actions/install-nix-action
with:
dogfood: false
extra_nix_config: |
experimental-features = flakes nix-command
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
- run: nix build .#dockerImage -L
- run: docker load -i ./result/image.tar.gz
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
#
# Push to Docker Hub first
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
- run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
# Push to GitHub Container Registry as well
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push image
run: |
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
# Change all uppercase to lowercase
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
docker tag nix:$NIX_VERSION $IMAGE_ID:latest
docker push $IMAGE_ID:$NIX_VERSION
docker push $IMAGE_ID:latest
# deprecated 2024-02-24
docker tag nix:$NIX_VERSION $IMAGE_ID:master
docker push $IMAGE_ID:master
flake_regressions:
needs: tests
runs-on: ubuntu-24.04
@@ -213,21 +287,13 @@ jobs:
with:
repository: NixOS/flake-regressions-data
path: flake-regressions/tests
- name: Download installer tarball
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
- uses: ./.github/actions/install-nix-action
with:
name: installer-linux
path: out
- name: Looking up the installer tarball URL
id: installer-tarball-url
run: |
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }}
- name: Run flake regressions tests
run: MAX_FLAKES=25 flake-regressions/eval-all.sh
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
extra_nix_config:
experimental-features = nix-command flakes
github_token: ${{ secrets.GITHUB_TOKEN }}
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
profile_build:
needs: tests
-69
View File
@@ -1,69 +0,0 @@
name: Upload Release
on:
workflow_dispatch:
inputs:
eval_id:
description: "Hydra evaluation ID"
required: true
type: number
is_latest:
description: "Mark as latest release"
required: false
type: boolean
default: false
permissions:
contents: read
id-token: write
packages: write
jobs:
release:
runs-on: ubuntu-24.04
environment: releases
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: ./.github/actions/install-nix-action
with:
dogfood: false # Use stable version
use_cache: false # Don't want any cache injection shenanigans
extra_nix_config: |
experimental-features = nix-command flakes
- name: Set NIX_PATH from flake input
run: |
NIXPKGS_PATH=$(nix build --inputs-from .# nixpkgs#path --print-out-paths --no-link)
# Shebangs with perl have issues. Pin nixpkgs this way. nix shell should maybe
# get the same uberhack that nix-shell has to support it.
echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
with:
role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
role-session-name: nix-release-oidc-${{ github.run_id }}
aws-region: eu-west-1
- name: Login to Docker Hub
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Upload release
run: |
./maintainers/upload-release.pl \
${{ inputs.eval_id }} \
--skip-git
env:
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
- name: Push to GHCR
run: |
DOCKER_OWNER="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[A-Z]' '[a-z]')/nix"
./maintainers/upload-release.pl \
${{ inputs.eval_id }} \
--skip-git \
--skip-s3 \
--docker-owner "$DOCKER_OWNER"
env:
IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
@@ -27,7 +27,7 @@ site](https://en.wikipedia.org/wiki/Call_site) position and the name of the
function being called (when available). For example:
```
/nix/store/2q71fdvr4h33g9832hiriwnf20fn630l-source/pkgs/top-level/default.nix:167:5:primop import
/nix/store/x9wnkly3k1gkq580m90jjn32q9f05q2v-source/pkgs/top-level/default.nix:167:5:primop import
```
Here `import` primop is called at `/nix/store/2q71fdvr4h33g9832hiriwnf20fn630l-source/pkgs/top-level/default.nix:167:5`.
Here `import` primop is called at `/nix/store/x9wnkly3k1gkq580m90jjn32q9f05q2v-source/pkgs/top-level/default.nix:167:5`.
@@ -39,11 +39,11 @@ This makes all subscribed channels available as attributes in the default expres
A symlink that ensures that [`nix-env`] can find the current user's [channels]:
- `~/.nix-defexpr/channels`
- `$XDG_STATE_HOME/nix/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`.
- `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`.
This symlink points to:
- `$XDG_STATE_HOME/nix/profiles/channels` for regular users
- `$XDG_STATE_HOME/profiles/channels` for regular users
- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root`
In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.
@@ -114,9 +114,9 @@ Here is an example of how this file might look like after installing `hello` fro
};
name = "hello-2.12.1";
out = {
outPath = "/nix/store/src1vzij2z0slnakrsbpqpk20389z0k6-hello-2.12.1";
outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1";
};
outPath = "/nix/store/src1vzij2z0slnakrsbpqpk20389z0k6-hello-2.12.1";
outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1";
outputs = [ "out" ];
system = "x86_64-linux";
type = "derivation";
@@ -37,13 +37,13 @@ dr-xr-xr-x 4 root root 4096 Jan 1 1970 share
/home/eelco/.local/state/nix/profiles/profile-7-link/bin:
total 20
lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/cyxny9d1zjb9l9103fr6j6kavp3bqjxf-chromium-86.0.4240.111/bin/chromium
lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications:
total 12
lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/sqzyx2l85i6j2a77pnyvglh3bvzwmjjp-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
@@ -72,11 +72,11 @@ When using public key authentication, you can avoid typing the passphrase with `
> $ storePath="$(nix-build '<nixpkgs>' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)"
> $ nix-copy-closure --to alice@itchy.example.org "$storePath"
> copying 5 paths...
> copying path '/nix/store/h6q8sqsqfbd3252f9gixqn3z282wds7m-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/imnwvn96lw355giswsk36hx105j4wnpj-libunistring-1.1' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/85301indj7scg34spnfczkz72jgv8wa9-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/ypwfsaljwhzw9iffiysxmxnhjj8v7np0-glibc-2.39-31' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/0dklv59zppdsqdvgf0qdvjgzcs5wbwxa-hello-2.12.1' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/nrwkk6ak3rgkrxbqhsscb01jpzmslf2r-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/gm61h1y42pqyl6178g90x8zm22n6pyy5-libunistring-1.1' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/ddfzjdykw67s20c35i7a6624by3iz5jv-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/apab5i73dqa09wx0q27b6fbhd1r18ihl-glibc-2.39-31' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/g1n2vryg06amvcc1avb2mcq36faly0mh-hello-2.12.1' to 'ssh://alice@itchy.example.org'...
> ```
> **Example**
@@ -204,7 +204,7 @@ To install a specific [store derivation] (typically created by
`nix-instantiate`):
```console
$ nix-env --install /nix/store/8la6y31fmm6i4wfmby6avly1wf718xnj-gcc-3.4.3.drv
$ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv
```
To install a specific output path:
@@ -232,7 +232,7 @@ $ nix-env --file '<nixpkgs>' --install --attr hello --dry-run
(dry run; not doing anything)
installing ‘hello-2.10’
this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
/nix/store/ikwkxz4wwlp2g1428n7dy729cg1d9hin-hello-2.10
/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
...
```
@@ -76,7 +76,7 @@ $ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz
```console
$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz
0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i
/nix/store/8alrpdaasjd1x6g1fczchmzbpqm936a3-hello-2.10.tar.gz
/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
```
```console
@@ -34,6 +34,6 @@ This operation has the following options:
```console
$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz
/nix/store/8alrpdaasjd1x6g1fczchmzbpqm936a3-hello-2.10.tar.gz
/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz
```
@@ -27,7 +27,7 @@ paths in the store that refer to it (i.e., depend on it).
# Example
```console
$ nix-store --delete /nix/store/gjak3al7lj61x4gj6rln4f5pc5v0f67n-mesa-6.4
$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4
0 bytes freed (0.00 MiB)
error: cannot delete path `/nix/store/gjak3al7lj61x4gj6rln4f5pc5v0f67n-mesa-6.4' since it is still alive
error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive
```
@@ -184,9 +184,9 @@ Print the build-time dependencies of `svn`:
```console
$ nix-store --query --requisites $(nix-store --query --deriver $(which svn))
/nix/store/y6qa66l9h0pw161crnlk6y16rdrcljx4-grep-2.5.1.tar.bz2.drv
/nix/store/z716h753s97jhnzvfank2srqbljswpgm-gcc-wrapper.sh
/nix/store/f39x0q73rjdyvzm93y9wrkfr6x39lb7f-glibc-2.3.4.drv
/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv
/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh
/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv
... lots of other paths ...
```
@@ -199,10 +199,10 @@ Show the build-time dependencies as a tree:
```console
$ nix-store --query --tree $(nix-store --query --deriver $(which svn))
/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv
+---/nix/store/vxnmkc8l8d2ijjha4xwhkfgx9vvc3q4c-builder.sh
+---/nix/store/rn9776dy82n5qrgz7xbcl1iw4vfkcrkk-bash-3.0.drv
| +---/nix/store/x9j20hz6bln1crzn55qifk0bbsm8v5ac-bash
| +---/nix/store/ajnn1mcm45wjvn0rlc22gvx2cwhjnazx-builder.sh
+---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh
+---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv
| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash
| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh
...
```
@@ -76,7 +76,7 @@ This operation is typically used to build [store derivation]s produced by
```console
$ nix-store --realise $(nix-instantiate ./test.nix)
/nix/store/6gwmy5jcnwdlz6aqqhksz863f1l8xc2w-aterm-2.3.1
/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1
```
This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does.
+1 -1
View File
@@ -136,7 +136,7 @@
> **Example**
>
> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1`
> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
See [Store Path](@docroot@/store/store-path.md) for details.
+1 -1
View File
@@ -8,7 +8,7 @@ stores packages in the _Nix store_, usually the directory
`/nix/store`, where each package has its own unique subdirectory such
as
/nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1/
/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/
where `b6gvzjyb2pg0…` is a unique identifier for the package that
captures all its dependencies (it’s a cryptographic hash of the
+2 -2
View File
@@ -34,12 +34,12 @@ String context elements come in different forms:
> [`builtins.storePath`] creates a string with a single constant string context element:
>
> ```nix
> builtins.getContext (builtins.storePath "/nix/store/ikwkxz4wwlp2g1428n7dy729cg1d9hin-hello-2.10")
> builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10")
> ```
> evaluates to
> ```nix
> {
> "/nix/store/ikwkxz4wwlp2g1428n7dy729cg1d9hin-hello-2.10" = {
> "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = {
> path = true;
> };
> }
@@ -181,7 +181,7 @@ A derivation interpolates to the [store path] of its first [output](./derivation
> "${pkgs.hello}"
> ```
>
> "/nix/store/qnlr7906z0mrl2syrkdbpicffq02nw07-hello-2.12.1"
> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1"
An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself.
+1 -23
View File
@@ -51,7 +51,6 @@ See [String literals](string-literals.md).
Path literals can also include [string interpolation], besides being [interpolated into other expressions].
[string interpolation]: ./string-interpolation.md
[interpolated into other expressions]: ./string-interpolation.md#interpolated-expression
At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path.
@@ -273,7 +272,7 @@ will crash with an `infinite recursion encountered` error message.
A let-expression allows you to define local variables for an expression.
> *let-in* = `let` [ *identifier* = *expr* `;` ]... `in` *expr*
> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr*
Example:
@@ -286,27 +285,6 @@ in x + y
This evaluates to `"foobar"`.
There is also another, older, syntax for let expressions that should not be used in new code:
> *let* = `let` `{` *identifier* = *expr* `;` [ *identifier* = *expr* `;`]... `}`
In this form, the attribute set between the `{` `}` is recursive.
One of the attributes must have the special name `body`,
which is the result of the expression.
Example:
```nix
let {
foo = bar;
bar = "baz";
body = foo;
}
```
This evaluates to "baz".
## Inheriting attributes
When defining an [attribute set](./types.md#type-attrs) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes).
@@ -94,8 +94,8 @@ properties:
>
> ```json
> "srcs": [
> "b8nwz167km1yciqpwzjj24f8jcy8pq1h-separate-debug-info.sh",
> "ihzmilr413r8fb3ah30yjnhlb18c1laz-fix-pop-var-context-error.patch"
> "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh",
> "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch"
> ]
> ```
items:
@@ -140,7 +140,7 @@ properties:
description: |
Absolute path of the program used to perform the build.
Typically this is the `bash` shell
(e.g. `/nix/store/p4xlj4imjbnm4v0x5jf4qysvyjjlgq1d-bash-4.4-p23/bin/bash`).
(e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
args:
type: array
+1 -1
View File
@@ -80,7 +80,7 @@
...
the following paths will be downloaded/copied (30.02 MiB):
/nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
/nix/store/spc1m987vlibchdx369qwa391s738s7l-libunwind-0.98.6
/nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
...
- Language features:
+1 -1
View File
@@ -63,7 +63,7 @@ Nix 0.8 has the following improvements:
can query all paths that directly or indirectly use a certain Glibc:
$ nix-store -q --referrers-closure \
/nix/store/1a6mdrjz4wn7b9sfmcw5ggbk1mi281mh-glibc-2.3.4
/nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
- The concept of fixed-output derivations has been formalised.
Previously, functions such as `fetchurl` in Nixpkgs used a hack
+2 -2
View File
@@ -66,7 +66,7 @@ This release has the following new features:
nix copy --to ssh://machine nixpkgs.hello
nix copy --to ssh://machine /nix/store/qbhyj3blxpw2i6pb7c6grc9185nbnpvy-hello-2.10
nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)'
@@ -187,7 +187,7 @@ This release has the following new features:
former is primarily useful in conjunction with remote stores,
e.g.
nix ls-store --store https://cache.nixos.org/ -lR /nix/store/qbhyj3blxpw2i6pb7c6grc9185nbnpvy-hello-2.10
nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
lists the contents of path in a binary cache.
+1 -1
View File
@@ -25,7 +25,7 @@
* Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables.
For example,
```shell-session
# nix build /nix/store/fpq78s2h8ffh66v2iy0q1838mhff06y8-glibc-2.33-78.drv^dev
# nix build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev
```
now works just as
```shell-session
+2 -2
View File
@@ -18,13 +18,13 @@
For example,
```shell-session
$ nix path-info /nix/store/fpq78s2h8ffh66v2iy0q1838mhff06y8-glibc-2.33-78.drv
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv
```
now gives info about the derivation itself, while
```shell-session
$ nix path-info /nix/store/fpq78s2h8ffh66v2iy0q1838mhff06y8-glibc-2.33-78.drv^*
$ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*
```
provides information about each of its outputs.
+2 -2
View File
@@ -45,7 +45,7 @@
```json5
[
{
"path": "/nix/store/fvqsvk65d38p8qqir371ii0hyqxvjcw6-bash-5.2-p15",
"path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15",
"valid": true,
// ...
},
@@ -60,7 +60,7 @@
```json5
{
"/nix/store/fvqsvk65d38p8qqir371ii0hyqxvjcw6-bash-5.2-p15": {
"/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": {
// ...
},
"/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null,
+1 -1
View File
@@ -182,7 +182,7 @@
«partially applied primop map»
nix-repl> builtins.trace lib.id "my-value"
trace: «lambda id @ /nix/store/kgr5lnaiiv08wb7k324yv1i1npjmrvjc-source/lib/trivial.nix:26:5»
trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5»
"my-value"
```
+3 -3
View File
@@ -2,7 +2,7 @@
> **Example**
>
> `/nix/store/jf6gn2dzna4nmsfbdxsd7kwhsk6gnnlr-git-2.38.1`
> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
>
> A rendered store path
@@ -22,7 +22,7 @@ Store paths are pairs of
> **Example**
>
> - Digest: `q06x3jll2yfzckz2bzqak089p43ixkkq`
> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z`
> - Name: `firefox-33.1`
To make store objects accessible to operating system processes, stores have to expose store objects through the file system.
@@ -38,7 +38,7 @@ A store path is rendered to a file system path as the concatenation of
> **Example**
>
> ```
> /nix/store/q06x3jll2yfzckz2bzqak089p43ixkkq-firefox-33.1
> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1
> |--------| |------------------------------| |----------|
> store directory digest name
> ```
+1 -1
View File
@@ -8,7 +8,7 @@ Stores are specified using a URL-like syntax. For example, the command
```console
# nix path-info --store https://cache.nixos.org/ --json \
/nix/store/1542dip9i7k4f24y6hqgd04hmvid9hr5-coreutils-9.1
/nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1
```
fetches information about a store path in the HTTP binary cache
+4
View File
@@ -418,6 +418,10 @@
supportsCross = false;
};
"nix-kaitai-struct-checks" = {
supportsCross = false;
};
"nix-perl-bindings" = {
supportsCross = false;
};
-32
View File
@@ -1,32 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
set -x
git ls-files -z \
| xargs -0 grep -o '[0123456789abcdfghijklmnpqrsvwxyz]\{32\}' 2> /dev/null \
| rev \
| cut -d: -f1 \
| rev \
| sort \
| uniq \
| while read -r oldhash; do
if ! curl --fail -I "https://cache.nixos.org/$oldhash.narinfo" > /dev/null 2>&1; then
continue
fi
newhash=$(
nix eval --expr "builtins.toFile \"006c6ssvddri1sg34wnw65mzd05pcp3qliylxlhv49binldajba5\" \"$oldhash\"" \
| cut -d- -f1 \
| cut -d/ -f4
)
msg=$(printf "bad: %s -> %s" "$oldhash" "$newhash")
echo "$msg"
git ls-files -z \
| xargs -0 grep -a -l "$oldhash" 2> /dev/null \
| while read -r file; do
[ -L "$file" ] && continue
perl -pi -e "s/$oldhash/$newhash/g" "$file" || true
done || true
git commit -am "$msg"
done
@@ -1,110 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBGPtMiwBEAC0sFZW2QW/OaDjKm5zGRpDvHXDsMIUtlHfoi5ce8pocC63W05o
FSXbUZjZ1VfYO8lT8DFANCzTkiXYaZx0cPRG2pVY4AOQZDNFt5XrAyvw496XCAIM
DTYGFLjCqgjPt9RUFEy4MyHPJTEpB0x3rXgT4ILNu9vsj9Q0vttps7SpbZ3Ldq5H
o/BBbLW77q/vNjpYzCbBIXF7ycUGpnNv9Go/WuiDnrBMcyxh+8kjjIHB5cxZSnjJ
DUv681+m83v+gLZQGX/jexQrrf5JpS0X9qEnhGLrNUDhtyv5ud3Je4EfamkjLVVC
RlNLofgflOCsl/tP80i+K7S1QdKhUALxuJ6H0prYUflGBDxDyC8XYuJ62TT0OUpa
vJvgwVlCq8/jq+ykYQXlbuBVOzi5wAuI4l3+HqreSQYPSiwe+6N590Zbafdv1fvN
WFtZKCTGMqfyaaAnppioH9/+NWkI2AQxaYVasYM/JEYvY9pJgA7alh51jHW4JglP
ErypKfBKPKJID0QENqYoa3bDDCihuNWhgQf9dxzPlj2ckd35Zb6w4DfuSmtjaa9D
o0jZVY1JbFuxBqP09+saVPrxLHgmPxjcdzPGQQtAqdO2vyJXNEGLFMoVEZPNaLo3
QmcIJnT7oSck+4vGfOYtWUHXQynu/Tnwsv2XkA/uyw8HNe+RRMqv/apnzQARAQAB
tCdTZXJnZWkgWmltbWVybWFuIDxzZXJnZWlAemltbWVybWFuLmZvbz6JAlEEEwEK
ADsCGwEFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQQVim9TDqIC5fZRYRMU+upj
RI4d+QUCaUbsaAIZAQAKCRAU+upjRI4d+VdDD/492HRaJ/8V7R7VUzkafmb2Hb28
SLf7oiB8Uq9I7SukiDEaIT1fUhquYWQ9KWpPRNR1TX6ApXnIeuJRMGFoDVIRnmnr
cKnYYXfqqc81VxIyKvaumB7KWbS7G4Nbor8AH1ouOOOMMS50OTJOWQA4A26inIuG
n+7L8MeS5aT+3uNKDoTKsidC47vnaxNMcke1taPfbfo7vn69PsRCM/g9/7TQYU8b
6xp+pM9Ao9nJneRk2YCpsGYRrWTpaik0DFKnfpPKJM/yunhtGLF2IYAp3l1mvHPK
nnzo92zjpQuZwazEIK+23V1vRT4IjM2BewbJPAzf2/UuxEjjgNQm0tOtH2JhFNeB
VM0BVrGxWrrwrsmv6lWghTtBc6zRWyHrj/rpjtVQNmeKYrHWJeXwVz1rqgGPmB2N
k0MZD1UjHHhEs1Cntn7yLmxTPztRJCtR+euRu81Uo2NAvrMJ4xsDjaM0LeLTnzjV
9AsPjD188dOFyz7VExZum4+XaaEJ41FIPLEqU3U0GAa6stEy0ylSlIN4x9aiXXVW
xfzHHchS5jK6QAjuZxN8t01GRactNylINRf7uoTECFZTtXNqfeuk0HQxBH0LuKVE
0PxJbcNI4mVWw1KgTJ8PUVC1IXP3sEpqPdJOYiRXgnpcS26fWOBu0aQ4mhxiaJhr
/zBfrkLEqp20TdNDZLkCDQRj7TM1ARAAt73xO24curnHTTgXkkVMMRzcMLx3Mb1a
2FuddxC5hzTpEpw01L91UBrXVJEg9K2KAwP5CtCLgPCqXr47Tm7krvHxWwBksgY/
6aHRsoPQfCFUZHc0aiO+C4NCzR+aEeGKn66Oc1Hq9oUTpDgiBWhsuEPiyA1OSGF0
4L0jeTCqfm68kWp4PIK9yuugkdDsoyj6TonuMsb3V5ctHLqop9KH+eHSkUTPo+Lk
+bxaeAOJ1UfbohgbRbrYKAfsaghhOMDH3R1w2pvtUJz+sDbuQsiPFTqbxsXDTFws
H4N/AQCYnnvOhqEek2sOEZ19bJXt5UrAr10mX4PGmAkWqE1JWBxpOKG3BXSGOTu1
3dFhQfPMK+PmvUrs0kcWQr53K/aRUdKKhIfTcMfkqYTGPK5HclHph24WjXj3QFFA
SjksQTdm6486ZmLZK4CTbAFOPfTF/aWg8gu9v4ihdq6lqHNNXxv2xBAChcd59H7p
D5zy9z8SpwWR9V5JDmlF6HWIIau1c6lSsQq1xHvYM8EuPe03vJvor+2u/cn5zYF1
5ZxAuPI2i5vtavg1s8ZGAAogJ9dVcP36LdJfL9quXWvmovkd//qHIepBB+l/zQio
ZRDZlIcfV3Xycaqsb5OqHGARHE0097koipMt5y/iXlqG4Ruue6Idb8bW96EKpaWj
kKy/iNfQfQMAEQEAAYkEcgQYAQoAJgIbAhYhBBWKb1MOogLl9lFhExT66mNEjh35
BQJpRuz3BQkJHCDCAkDBdCAEGQEKAB0WIQRK3hK0WyJ4BicGpcmpsLVXymMjJQUC
Y+0zNQAKCRCpsLVXymMjJbdSD/9+f1FOOeGDAJI6Duo5fsWnf4xJJdtQtDbz6d2A
SeDapxeJ3zWfKBD0wu5sISEa0uiWsYSmLtsa2SqVAKHlEaMGRR+tkBMPQ+rvgI4c
62YjGTgm+IPd+NFIn+ixFU1hpinTh+KhUEoeOwWCvKs9nZfSG9vkienfiG0bBxo2
zrvBzXA50x5hbUL+ghKu/AVfN9qZDwh30O4KZTwk4g4cM9SeaQa4YvHYIS3IEhDZ
hGybwrrqV9cs92ln4IJw9WCy9QReBNrdeFgC4+3ziUp1QsG3RvqrtuMttwBVC1Z3
bj5QjLLOREhhodfvk98t9yVkragObb4rGrLo1mWuF0c4mJGvXwnrqhCMvzv4M+0T
Zdrmw6YpGkGOaOPghVuwoTtqSAkl+zFWIJS89jidvkYG3EqKAkgLKog/TQReCq13
HWrF8cMck+Rf2K8k26q/RNZaA9ZUKjLExzz8lsWmd2C7rvkGLrlxnzxz0gGyNR3Q
KK74vcPhqeABt2GSkHtEXZFFA9IVVzwlRWK3e0S+mVQnZVjNL+cBPn3/hZHMLesB
CucyYZv+DxvT+JkYXBkGSw4s3hpABqGym7gdPUIa0q4rbBFG6xP5sLLBG4yru8vV
2dyCMmFqRuxpT49uNfyQ6Vj+dobN6qHnP/9NwfzOixXYBHXR6LBqb/M+iCiJaaIn
uiRLHwkQFPrqY0SOHfkQpA//W51vj8meuz7snRO+vZFcjLneFFzqfh1Jdz8IqDpO
CkI5pBJmi8e0oSe6r68MkahiQLlYPwm7d+sjHvJhPWipNKWq/uwCgBs+Ac1lpPXR
MwLbrZukcLMYlLmb2MrCKmjcMt0BZsZKBNYL3a3X9nHgwXdeqFYS4WQDMCCc09lz
9YqfdoEsqRO4qN7D0hFqnwjOzb34ixZ6UO8a8ekY9QKxAgWc9fJWGMg6Pjdg4qsK
nqymOIAdGVOJdoRM46wKGVBvbsF2gNfQU4XyzgJo5vHGFwJm6EoSnODlL5e2wsQh
uN1oqBt/8ef/plloMEqVBweUBATqSqjRF6IhhYJvWVuQHQL1p1vnV9FebiVj34ir
Z8ID+o0AnTJcclbUcDwannGJ0cuDcPhk/v/ahVuoMERCi12qnMBo5B/e6Omyh1yB
4pbf4GATGGQipDQG75eC/kP2GQEqJP5WYN0Ar8Le/AA/2xyL7upW0yIByyXCwGEb
JRwEgU3+bPyu58bFt8Pftit6J7rA3oBVVMOPrYH5eZwRaj5m2RptwKGL6BfHnhNv
ZqmCq9EBGX6L1NI0xHMjEFfXJ8jU01XdfG8nCqkwqsHwslXLhqjJphfHcx89YwbV
/15GCuURAv1cKe/7277sOhcvP/QpQqSWgvYExHw8PeFJcTYtF2NrRgNwcQsWS1Rj
gXa5Ag0EY+0zcAEQANC5N6kSfezuucAgi+X3BD+MT37mxQyvICSggEJf1LDSmy0+
bnvD7setL8CP9etTA2fcVNYKI1oboMyhoCnsRP2jDdv1iXOI/hZg4wSb/D1yUkae
fUpxv3Wuci2QKavH2MfraDD7BFMbsQeMcHtn4Rk216T6jndZHnzT1Ih7iX0XeQPb
li5fojOiZssgWAVT4HPXFCJB6lI35Hjp35oRYwrtMmu5INinZ79n9h1igGtt1ItZ
b7rQKNd772Jxcn4UU71ovORSL/xT5i5sxZ+evQOxkpqUAokMOFaoHcOXLmA1NsFv
yryXHK4Ioq9ap2jKlLTWkJWjua9JZ4AmKhbvT8X4ELxIKSCAdJKAWP8ZHbXNu5MD
aznyzZQLxSO7uFvu356De75mI5iohZNj5wB5Wju71pBiorTKVj4+iJ4e+xVIzFdG
hFC0DehNcl2t9w/y8qHwIQ1yUAjXHLXq0/2jsVeH6bU5q/MsgvUP1jcFe0eyOpxy
CDvyFdzZFbI57TnB/fvcZTRZ5ewXMFpH8gzuoFzAjUAP95UjYKgaGdrNPNIy28Ii
4zhvdghei2+n9jgiMfcGQg8lyfH5yF0vWWWynX0KcJsRwEZoL2EauVdwq4PcYOoU
pQFhpcreCjD4LdZ4yRU4InbhcUogXjrQ9Dz01TbPmQD5b5iso21bCEFBXrhzABEB
AAGJAjwEGAEKACYCGwwWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwg
hwAKCRAU+upjRI4d+X/XD/sH5xvHPfTJq52v8weFmB52up+DzqG2lyhGdoUQ1Muw
dRDLTLXLJrFdfpoOo7/j4Scr0rdc7/dpCn0DLcPuCoPxu+SkjEnVehFmZrGSv7Ga
x9dHr3DBh42fdlX/U/EnDuyosY0JU1gNF2/6FIA+bTTOFE3RxfN906RjslYQDjMZ
UAlSeLYHOZofdltI0YIr32vrxgdWQGZXPxU4XusDUc0z163OO+TGg7iUNWFZP5Qj
ubM7e0YbDX0NPIshk8us99YJmrWnhaix1/W5ryO3DXiGaQ7XFi9u7QofRqvRIctg
QXavdepkzJow9V9qpMECAJePIuICq7rm+xy+njjbuF436W7390bfVBwRr+FPADsl
jgQP4KvY5rykss30kheom8wNEbveWkhH5oTfH9b7O4KXJfpfJzrlgOWp2BD9JL8t
/M4HvFXTr2a75H/QbHK5OFrZeGATuv9OTxv7EZvnrPXU+DYTFldpu7TrNNqKCoj3
ZyXmc3Hhg5kskDhfHJppaeOayuhMOpT3ud1MFzROY5SLVIH8rBR12KUgsCUYQcGs
Iy0+0QvEGkjb4cAH1NK3VlbqVNsy1RmqRt2B28R2ueewDfTOoqkzt4MmzLqTdnAx
mTqmHmkEKhEf3K4MRNUPO2yieUg2COk5l6x9HhAnoxxeOZrTmcMsPY/UViG2HEPm
ybkCDQRj7TRDARAA9DZuKdfKq4Bs2+NwxC0aplljWOl8VIsEVg+Q8agD7/HU6/b6
Dry0njtWybn2x6Axf/nUdeOC01Fi1lmht/fpj6mRkgAvd/V6P10xnsUoykPSDSTh
P25MFFGW3JAA82bwdJ4AJpEQvTZG2nTb3237vlBiI1qHQrac8GYkju2O4UfySRN6
7cyi7bMf2pjWBBOEhaNy4b6CMDsb32P/N5J7sTE/TXgrS+u4ITIgjzSrkUkh5Z+B
8QVRa7xPIDZJdvZWTEXWu5fgRPZvxbr154GIkWJkFzlDoB1UcO56/uzRUuKhEV6o
HW3LMUuWdPMjpHpq8hrL0G2rDniJFUtbDFzHdZK1LUU3T2BJM8rjI3D/euph+IDT
27vl5qo72zCYE/iKzx4FMLZcQvx1kUAxkPX8l+dzZEwKeRIIpFDxQvatRtl+z0bM
jbkpDb+Yjv66sC4dYRpgTTGX6rok0PWHR3IxDNzyf2j8zQ4LFJ+rVBM1GjGSt6mG
j9TeL8CVeiSp4SuJ7I/FJVPHsKb50m+BDzeB31qTydNqh2kKr0DVAUa+TUsCr7e0
OYr8WE2adJcRXIW0qw50xXF+W7/05GqSCVD0dpeOUdBTQTsSkQmM3/0hcj9aVo9e
UDCM9RF0WRqiDAoHzJFfg+ztamkQI5HO6CklC4Ok22qrHRf6HDNYSuT6QFkAEQEA
AYkCPQQYAQoAJwMbIAQWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwf
tAAKCRAU+upjRI4d+Y+cD/9yllG6uo934pcHNsVppZBfREFwSc8ywlbosCuSVpay
PjSqgrWwDrnqrsk0F2kUdC6rR3BIcXbn+lA9KqylH+cCXAJCkh8EDq6TlQ7Lt5EV
w1U0MAMXOyxPwDymQ/BO+iDyjXWkRRYgbF5XiFhCfGeuKyhkhACisAgNZ1uA1P5k
0SJYc14YfEhQkB46Y20SpfVHRsQ46FyNB6GHbmTmfoO8La8VTh++7GBdh85HfvkG
VNQ3wpi5oXsOLN9+MJOezc0XsW2LQsKQj1/J7QKzGh+lxN5cemsA5aqPzh8dyxeT
0lYRFp4AHkimqGUomVpRkbegMIPxXqOE+ZAmsddErw0UtmrKxcmMptOJwNgYzEgu
++2vtqerL/NYp+wsdcWaBjCz2F3NiwHgNli7NSB/FPwucZZ5gN5C4SnmeFzrGdHg
Oy+tQUN6ayQKljHeBO7CjMlsFNo/dcVrEMa1ShxBMqlj/6ivoEhktLz0Nru4FwNU
xE5SJYDYfpjD7Ws8y4LoXgWXjFHrMO6N9GzqLN/e8LT7I+w4ps2MrgJ8QSrelmQ3
rjkxp3uWp5v2lqy4rLfpi9iB6zIAeoN2eU1yOM9joxOYMxKYaYeYyP1Mm90wFol8
LcTSaN+tVniPddBiL6zvsGBEMbCR9XN3EQ+mErbuw5ovWBOCrr+dvN3FxvD11y4J
7w==
=mXYP
-----END PGP PUBLIC KEY BLOCK-----
@@ -1,51 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQENBFZu2zwBCADfatenjH3cvhlU6AeInvp4R0JmPBG942aghFj1Qh57smRcO5Bv
y9mqrX3UDdmVvu58V3k1k9/GzPnAG1t+c7ohdymv/AMuNY4pE2sfxx7bX+mncTHX
5wthipn8kTNm4WjREjCJM1Bm5sozzEZetED3+0/dWlnHl8b38evnLsD+WbSrDPVp
o6M6Eg9IfMwTfcXzdmLmSnGolBWDQ9i1a0x0r3o+sDW5UTnr7jVP+zILcnOZ1Ewl
Rn9OJ4Qg3ULM7WTMDYpKH4BO7RLR3aJgmsFAHp17vgUnzzFBZ10MCS3UOyUNoyph
xo3belf7Q9nrHcSNbqSeQuBnW/vafAZUreAlABEBAAG0IkVlbGNvIERvbHN0cmEg
PGVkb2xzdHJhQGdtYWlsLmNvbT6JATwEEwEIACYCGyMHCwkIBwMCAQYVCAIJCgsE
FgIDAQIeAQIXgAUCVm7etAIZAQAKCRCBcLRybXGY3q51B/96qt41tmcDSzrj/UTl
O6rErfW5zFvVsJTZ95Duwu87t/DVhw5lKBQcjALqVddufw1nMzyN/tSOMVDW8xe4
wMEdcU4+QAMzNX80enuyinsw1glxfLcK0+VbTvqNIfw0sG3MjPqNs6cK2VRfMHK4
paJjytBVICszNX9TfjLyIpKKoSSo1vqnT47LDZ5GIMy7l9Cs2sO/rqQHSPcR79yz
8m8tbHpDDEMZmJeklckKP2QoiqnHiIvlisDxLclYnUmNaPdaN/f++qZz5Yqvu1n+
sNUBA5eLaZH64Uy2SwtABxO3JPJ8nQ2+SFZ7ocFm4Gcdv4aM+Ura9S6fvM91tEJp
yAQOiQE5BBMBCAAjBQJWbts8AhsjBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AA
CgkQgXC0cm1xmN6sIAgAielxO8zJREqEkA2xudg/o4e9ZlNZ3X1NvY8OzJH/qlB2
SmwKqwifhtbC1K0uavXA7eaxdtd2zrI+Yq7IooUyv7juMjHTZhLcFbR5iVkQ4Mfp
JmeHXJ/ChYKxD5mMj/C3WbCZ91oCSNZ6Iyi5fvQj/691OC4q+y/2NEUcOI8D8cw8
XKHbKtceFYc+nZmdOv3ZZrNTSN/kszGViNNLKgnpPdDVPtLp+vjXtbmitiFG2HL/
WfbJ+3Gh2Yr1Vy3O9dWKH++e1AmIv7WWqmUjRFVpqC/wr7/BLaScWT8WKF5vkshU
gq8Ez1/cuizsgs3wQIZWgXKQK5njvwnbKg+Zmh/uGbQmRWVsY28gRG9sc3RyYSA8
ZWVsY28uZG9sc3RyYUB0d2VhZy5pbz6JAU4EEwEIADgWIQS1QdVTAScOC88Vyl2B
cLRybXGY3gUCXELt4gIbIwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCBcLRy
bXGY3ujFCADfS5D1xHU8KH6TpqgssSggYVq62Wwn/Ga+4XPPetM+ajcXagyH6SwB
mxlHICcnv9xC93ryiTI10P1ADJl+aBsI66wEdHBU+ty4RTDy4JZNUPtmRCk9LhSc
mtUO3ry/wtWkRLdJxP49hg7BbQvWoU0M6WODp7SJjPKPWNX64mzHBeOuy+DqGCbM
lpGNCvW8ahU/ewbm7+xwWmzqLDoWzXjHsdF4QdzMVM/vkAgWEP4y0wEqFASzIYaR
GNEkBWU4OQVq5Bdm9+wWWAgsbM0FJAQl0GDqnz4QxWzxxCAAXdbh9F5ffafWYsA9
bise4ZQLkvYo6iUnrcFm4dtZbT8iL3gptCtFZWxjbyBEb2xzdHJhIDxlZWxjby5k
b2xzdHJhQGxvZ2ljYmxveC5jb20+iQE5BBMBCAAjBQJWbt6nAhsjBwsJCAcDAgEG
FQgCCQoLBBYCAwECHgECF4AACgkQgXC0cm1xmN4b/wf8DApMV/jSPEpibekrUPQu
Ye3Z8cxBQuRm/nOPowtPEH/ShAevrCdRiob2nuEZWNoqZ2e5/+6ud07Hs9bslvco
cDv1jeY1dof1idxfKhH3kfSpuD2XJhuzQBxBqOrIlCS/rdnW+Y9wOGD7+bs9QpcA
IyAeQGLLkfggAxaGYQ2Aev8pS7i3a/+lOWbFhcTe02I49KemCOJqBorG5FfILLNr
DjO3EoutNGpuz6rZvc/BlymphWBoAdUmxgoObr7NYWgw9pI8WeE6C7bbSOO7p5aQ
spWXU7Hm17DkzsVDpaJlyClllqK+DdKza5oWlBMe/P02jD3Y+0P/2rCCyQQwmH3D
RbkBDQRWbts8AQgA0g556xc08dH5YNEjbCwEt1j+XoRnV4+GfbSJIXOl9joIgzRC
4IaijvL8+4biWvX7HiybfvBKto0XB1AWLZRC3jWKX5p74I77UAcrD+VQ/roWQqlJ
BKbiQMlRYEsj/5Xnf72G90IP4DAFKvNl+rLChe+jUySA91BCtrYoP75Sw1BE9Cyz
xEtm4WUzKAJdXI+ZTBttA2Nbqy+GSuzBs7fSKDwREJaZmVrosvmns+pQVG4WPWf4
0l4mPguDQmZ9wSWZvBDkpG7AgHYDRYRGkMbAGsVfc6cScN2VsSTa6cbeeAEowKxM
qx9RbY3WOq6aKAm0qDvow1nl7WwXwe8K0wQxfQARAQABiQEfBBgBCAAJBQJWbts8
AhsMAAoJEIFwtHJtcZjeuAAH/0YNz2Qe1IAEO5oqEZNFOccL4KxVPrBhWUen83/b
C6PjOnOqv6q5ztAcms88WIKxBlfzIfq+dzJcbKVS/H7TEXgcaC+7EYW8sJVEsipN
BtEZ3LQNJ5coDjm7WZygniah1lfXNuiritAXduK5FWNNndqGArEaeZ8Shzdo/Uyi
b9lOsBIL6xc2ZcnX5f+rTu02LCEtEb0FwCycZLEWYf8hG4k8uttIOZOC+CLk/k8d
kBmPikMwUVTTV0CdT1cemQKdTaoAaK+kurF6FYXwcnjhRlHrisSt/tVMEwTw4LUM
3MYf6qfjjvE4HlDwZal8th7ccoQp/flfJIuRv85xCcKK+PI=
=u5cX
-----END PGP PUBLIC KEY BLOCK-----
-13
View File
@@ -1,13 +0,0 @@
# Maintainer GPG Keys
Release tags are signed by members of the [Nix maintainer team](https://nixos.org/community/teams/nix/) as part of the [release process](../release-process.md). This directory contains the public GPG keys used for signing.
## Keys
- **Eelco Dolstra**
GPG Fingerprint: `B541 D553 0127 0E0B CF15 CA5D 8170 B472 6D71 98DE`
- **Sergei Zimmerman**
GPG Fingerprint: [`158A 6F53 0EA2 02E5 F651 6113 14FA EA63 448E 1DF9`](https://keys.openpgp.org/vks/v1/by-fingerprint/158A6F530EA202E5F651611314FAEA63448E1DF9)
<!-- TODO: Add keys for other Nix team members -->
+11 -9
View File
@@ -5,11 +5,11 @@
The release process is intended to create the following for each
release:
* A signed Git tag (public keys in `maintainers/keys/`)
* A Git tag
* Binary tarballs in https://releases.nixos.org/?prefix=nix/
* Docker images (arm64 and amd64 variants, uploaded to DockerHub and GHCR)
* Docker images
* Closures in https://cache.nixos.org
@@ -104,17 +104,21 @@ release:
evaluation ID (e.g. `1780832` in
`https://hydra.nixos.org/eval/1780832`).
* Tag the release:
* Tag the release and upload the release artifacts to
[`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/):
```console
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
```
Note: `IS_LATEST=1` causes the `latest-release` branch to be
force-updated. This is used by the `nixos.org` website to get the
[latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/).
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
TODO: This script requires the right AWS credentials. Document.
TODO: This script currently requires a
`/home/eelco/Dev/nix-pristine`.
TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
@@ -177,18 +181,16 @@ release:
* Wait for the desired evaluation of the maintenance jobset to finish
building.
* Tag the release
* Run
```console
$ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
$ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
```
Omit `IS_LATEST=1` when creating a point release that is not on the
most recent stable branch. This prevents `nixos.org` to going back
to an older release.
* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
* Bump the version number of the release branch as above (e.g. to
`2.12.2`).
+82 -128
View File
@@ -1,8 +1,7 @@
#! /usr/bin/env nix-shell
#! nix-shell -i perl -p awscli2 perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 perlPackages.GetoptLongDescriptive gnupg1
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
use strict;
use Getopt::Long::Descriptive;
use Data::Dumper;
use File::Basename;
use File::Path;
@@ -14,30 +13,7 @@ use Net::Amazon::S3;
delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
my ($opt, $usage) = describe_options(
'%c %o <eval-id>',
[ 'skip-docker', 'Skip Docker image upload' ],
[ 'skip-git', 'Skip Git tagging' ],
[ 'skip-s3', 'Skip S3 upload' ],
[ 'docker-owner=s', 'Docker image owner', { default => 'nixos/nix' } ],
[ 'project-root=s', 'Pristine git repository path' ],
[ 's3-endpoint=s', 'Custom S3 endpoint' ],
[ 's3-host=s', 'S3 host', { default => 's3-eu-west-1.amazonaws.com' } ],
[],
[ 'help|h', 'Show this help message', { shortcircuit => 1 } ],
[],
[ 'Environment variables:' ],
[ 'AWS_ACCESS_KEY_ID' ],
[ 'AWS_SECRET_ACCESS_KEY' ],
[ 'AWS_SESSION_TOKEN For OIDC' ],
[ 'IS_LATEST Set to "1" to mark as latest release' ],
);
print($usage->text), exit if $opt->help;
my $evalId = $ARGV[0] or do { print STDERR $usage->text; exit 1 };
die "--project-root is required unless --skip-git is specified\n" unless $opt->skip_git || $opt->project_root;
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
my $releasesBucketName = "nix-releases";
my $channelsBucketName = "nix-channels";
@@ -86,38 +62,25 @@ File::Path::make_path($narCache);
my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'};
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'};
my $aws_session_token = $ENV{'AWS_SESSION_TOKEN'};
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
my ($s3, $releasesBucket, $s3_channels, $channelsBucket);
my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});
unless ($opt->skip_s3) {
$aws_access_key_id or die "No AWS_ACCESS_KEY_ID given.";
$aws_secret_access_key or die "No AWS_SECRET_ACCESS_KEY given.";
my $releasesBucket = $s3->bucket($releasesBucketName) or die;
$s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
retry => 1,
host => $opt->s3_host,
secure => ($opt->s3_endpoint && $opt->s3_endpoint =~ /^http:/) ? 0 : 1,
});
my $s3_us = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
});
$releasesBucket = $s3->bucket($releasesBucketName) or die;
$s3_channels = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
$aws_session_token ? (aws_session_token => $aws_session_token) : (),
retry => 1,
$opt->s3_endpoint ? (host => $opt->s3_host) : (),
$opt->s3_endpoint ? (secure => ($opt->s3_endpoint =~ /^http:/) ? 0 : 1) : (),
});
$channelsBucket = $s3_channels->bucket($channelsBucketName) or die;
}
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
sub getStorePath {
my ($jobName, $output) = @_;
@@ -152,12 +115,11 @@ sub copyManual {
File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1});
}
my $awsEndpoint = $opt->s3_endpoint ? "--endpoint-url " . $opt->s3_endpoint : "";
system("aws $awsEndpoint s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
or die "syncing manual to S3\n";
}
copyManual unless $opt->skip_s3;
copyManual;
sub downloadFile {
my ($jobName, $productNr, $dstName) = @_;
@@ -196,12 +158,30 @@ sub downloadFile {
return $sha256_expected;
}
# Upload docker images.
downloadFile("binaryTarball.i686-linux", "1");
downloadFile("binaryTarball.x86_64-linux", "1");
downloadFile("binaryTarball.aarch64-linux", "1");
downloadFile("binaryTarball.x86_64-darwin", "1");
downloadFile("binaryTarball.aarch64-darwin", "1");
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
};
warn "$@" if $@;
downloadFile("installerScript", "1");
# Upload docker images to dockerhub.
my $dockerManifest = "";
my $dockerManifestLatest = "";
my $haveDocker = 0;
unless ($opt->skip_docker) {
for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
my $system = $platforms->[0];
my $dockerPlatform = $platforms->[1];
@@ -215,8 +195,8 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
print STDERR "loading docker image for $dockerPlatform...\n";
system("docker load -i $tmpDir/$fn") == 0 or die;
my $tag = $opt->docker_owner . ":$version-$dockerPlatform";
my $latestTag = $opt->docker_owner . ":latest-$dockerPlatform";
my $tag = "nixos/nix:$version-$dockerPlatform";
my $latestTag = "nixos/nix:latest-$dockerPlatform";
print STDERR "tagging $version docker image for $dockerPlatform...\n";
system("docker tag nix:$version $tag") == 0 or die;
@@ -239,94 +219,68 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
}
if ($haveDocker) {
my $dockerOwner = $opt->docker_owner;
print STDERR "creating multi-platform docker manifest...\n";
system("docker manifest rm $dockerOwner:$version");
system("docker manifest create $dockerOwner:$version $dockerManifest") == 0 or die;
system("docker manifest rm nixos/nix:$version");
system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
if ($isLatest) {
print STDERR "creating latest multi-platform docker manifest...\n";
system("docker manifest rm $dockerOwner:latest");
system("docker manifest create $dockerOwner:latest $dockerManifestLatest") == 0 or die;
system("docker manifest rm nixos/nix:latest");
system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
}
print STDERR "pushing multi-platform docker manifest...\n";
system("docker manifest push $dockerOwner:$version") == 0 or die;
system("docker manifest push nixos/nix:$version") == 0 or die;
if ($isLatest) {
print STDERR "pushing latest multi-platform docker manifest...\n";
system("docker manifest push $dockerOwner:latest") == 0 or die;
system("docker manifest push nixos/nix:latest") == 0 or die;
}
}
}
# Upload nix-fallback-paths.nix.
write_file("$tmpDir/fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
"}\n");
# Upload release files to S3.
unless ($opt->skip_s3) {
downloadFile("binaryTarball.i686-linux", "1");
downloadFile("binaryTarball.x86_64-linux", "1");
downloadFile("binaryTarball.aarch64-linux", "1");
downloadFile("binaryTarball.x86_64-darwin", "1");
downloadFile("binaryTarball.aarch64-darwin", "1");
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
};
warn "$@" if $@;
eval {
downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
};
warn "$@" if $@;
downloadFile("installerScript", "1");
for my $fn (glob "$tmpDir/*") {
my $name = basename($fn);
next if $name eq "manual";
my $dstKey = "$releaseDir/" . $name;
unless (defined $releasesBucket->head_key($dstKey)) {
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
# Upload nix-fallback-paths.nix.
write_file("$tmpDir/fallback-paths.nix",
"{\n" .
" x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
" i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
" aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
" riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
" x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
" aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
"}\n");
my $configuration = ();
$configuration->{content_type} = "application/octet-stream";
for my $fn (glob "$tmpDir/*") {
my $name = basename($fn);
next if $name eq "manual";
my $dstKey = "$releaseDir/" . $name;
unless (defined $releasesBucket->head_key($dstKey)) {
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
my $configuration = ();
$configuration->{content_type} = "application/octet-stream";
if ($fn =~ /.sha256|install|\.nix$/) {
$configuration->{content_type} = "text/plain";
}
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
if ($fn =~ /.sha256|install|\.nix$/) {
$configuration->{content_type} = "text/plain";
}
}
# Update the "latest" symlink.
$channelsBucket->add_key(
"nix-latest/install", "",
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
or die $channelsBucket->err . ": " . $channelsBucket->errstr
if $isLatest;
$releasesBucket->add_key_filename($dstKey, $fn, $configuration)
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
}
}
# Update the "latest" symlink.
$channelsBucket->add_key(
"nix-latest/install", "",
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
or die $channelsBucket->err . ": " . $channelsBucket->errstr
if $isLatest;
# Tag the release in Git.
unless ($opt->skip_git) {
chdir($opt->project_root) or die "Cannot chdir to " . $opt->project_root . ": $!";
system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
system("git push origin refs/tags/$version") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
}
chdir("/home/eelco/Dev/nix-pristine") or die;
system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
system("git push --tags") == 0 or die;
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
File::Path::remove_tree($narCache, {safe => 1});
File::Path::remove_tree($tmpDir, {safe => 1});
+3
View File
@@ -63,3 +63,6 @@ subproject('nix-functional-tests')
if get_option('json-schema-checks')
subproject('json-schema-checks')
endif
if get_option('kaitai-struct-checks')
subproject('kaitai-struct-checks')
endif
+7
View File
@@ -28,6 +28,13 @@ option(
description : 'Build benchmarks (requires gbenchmark)',
)
option(
'kaitai-struct-checks',
type : 'boolean',
value : true,
description : 'Check the Kaitai Struct specifications (requires Kaitai Struct)',
)
option(
'json-schema-checks',
type : 'boolean',
+6 -1
View File
@@ -133,7 +133,7 @@ let
+
lib.optionalString
(
!(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isCygwin)
!stdenv.hostPlatform.isWindows
# build failure
&& !stdenv.hostPlatform.isStatic
# LTO breaks exception handling on x86-64-darwin.
@@ -452,6 +452,11 @@ in
*/
nix-json-schema-checks = callPackage ../src/json-schema-checks/package.nix { };
/**
Kaitai struct schema validation checks
*/
nix-kaitai-struct-checks = callPackage ../src/kaitai-struct-checks/package.nix { };
nix-perl-bindings = callPackage ../src/perl/package.nix { };
/**
-1
View File
@@ -40,7 +40,6 @@ scope: {
postInstall =
lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
(prevAttrs.postInstall or "");
patches = [ ];
});
# TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
+2 -3
View File
@@ -142,9 +142,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
internalDrvs = byDrvPath (
# Drop the attr names (not present in buildInputs anyway)
lib.attrValues availableComponents
++ lib.concatMap (c: lib.filter (v: !v.meta.broken) (lib.attrValues (c.tests or { }))) (
lib.attrValues availableComponents
)
++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents)
);
isInternal =
@@ -282,6 +280,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
dontUseCmakeConfigure = true;
mesonFlags = [
(lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks"))
(lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
]
++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
+1
View File
@@ -72,6 +72,7 @@ let
"nix-manual-manpages-only"
"nix-internal-api-docs"
"nix-external-api-docs"
"nix-kaitai-struct-checks"
]
);
in
+2 -2
View File
@@ -53,8 +53,8 @@ readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_CACERT="@cacert@"
#readonly NIX_INSTALLED_NIX="/nix/store/byi37zv50wnfrpp4d81z3spswd5zva37-nix-2.3.6"
#readonly NIX_INSTALLED_CACERT="/nix/store/7pi45g541xa8ahwgpbpy7ggsl0xj1jj6-nss-cacert-3.49.2"
#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
EXTRACTED_NIX_PATH="$(dirname "$0")"
readonly EXTRACTED_NIX_PATH
+1
View File
@@ -0,0 +1 @@
../../.version
+77
View File
@@ -0,0 +1,77 @@
# Run with:
# meson test --suite kaitai-struct
# Run with: (without shell / configure)
# nix build .#nix-kaitai-struct-checks
project(
'nix-kaitai-struct-checks',
'cpp',
version : files('.version'),
default_options : [
'cpp_std=c++23',
# TODO(Qyriad): increase the warning level
'warning_level=1',
'errorlogs=true', # Please print logs for tests that fail
],
meson_version : '>= 1.1',
license : 'LGPL-2.1-or-later',
)
kaitai_runtime_dep = dependency('kaitai-struct-cpp-stl-runtime', required : true)
gtest_dep = dependency('gtest')
gtest_main_dep = dependency('gtest_main', required : true)
# Find the Kaitai Struct compiler
ksc = find_program('ksc', required : true)
kaitai_generated_srcs = custom_target(
'kaitai-generated-sources',
input : [ 'nar.ksy' ],
output : [ 'nix_nar.cpp', 'nix_nar.h' ],
command : [
ksc,
'@INPUT@',
'--target', 'cpp_stl',
'--outdir',
meson.current_build_dir(),
],
)
nar_kaitai_lib = library(
'nix-nar-kaitai-lib',
kaitai_generated_srcs,
dependencies : [ kaitai_runtime_dep ],
install : true,
)
nar_kaitai_dep = declare_dependency(
link_with : nar_kaitai_lib,
sources : kaitai_generated_srcs[1],
)
# The nar directory is a committed symlink to the actual nars location
nars_dir = meson.current_source_dir() / 'nars'
# Get all example files
nars = [
'dot.nar',
]
test_deps = [
nar_kaitai_dep,
kaitai_runtime_dep,
gtest_main_dep,
]
this_exe = executable(
meson.project_name(),
'test-parse-nar.cc',
dependencies : test_deps,
)
test(
meson.project_name(),
this_exe,
env : [ 'NIX_NARS_DIR=' + nars_dir ],
protocol : 'gtest',
)
+1
View File
@@ -0,0 +1 @@
../../doc/manual/source/protocols/nix-archive/nar.ksy
+1
View File
@@ -0,0 +1 @@
../libutil-tests/data/nars
+1
View File
@@ -0,0 +1 @@
../../nix-meson-build-support
+70
View File
@@ -0,0 +1,70 @@
# Run with: nix build .#nix-kaitai-struct-checks
# or: `nix develop .#nix-kaitai-struct-checks` to enter a dev shell
{
lib,
mkMesonDerivation,
gtest,
meson,
ninja,
pkg-config,
kaitai-struct-compiler,
fetchzip,
kaitai-struct-cpp-stl-runtime,
# Configuration Options
version,
}:
let
inherit (lib) fileset;
in
mkMesonDerivation (finalAttrs: {
pname = "nix-kaitai-struct-checks";
inherit version;
workDir = ./.;
fileset = lib.fileset.unions [
../../nix-meson-build-support
./nix-meson-build-support
./.version
../../.version
../../doc/manual/source/protocols/nix-archive/nar.ksy
./nars
../../src/libutil-tests/data
./meson.build
./nar.ksy
(fileset.fileFilter (file: file.hasExt "cc") ./.)
(fileset.fileFilter (file: file.hasExt "hh") ./.)
];
outputs = [ "out" ];
buildInputs = [
gtest
kaitai-struct-cpp-stl-runtime
];
nativeBuildInputs = [
meson
ninja
pkg-config
# This can go away when we bump up to 25.11
(kaitai-struct-compiler.overrideAttrs (finalAttrs: {
version = "0.11";
src = fetchzip {
url = "https://github.com/kaitai-io/kaitai_struct_compiler/releases/download/${version}/kaitai-struct-compiler-${version}.zip";
sha256 = "sha256-j9TEilijqgIiD0GbJfGKkU1FLio9aTopIi1v8QT1b+A=";
};
}))
];
doCheck = true;
mesonCheckFlags = [ "--print-errorlogs" ];
postInstall = ''
touch $out
'';
meta = {
platforms = lib.platforms.unix;
};
})
@@ -0,0 +1,48 @@
#include <gtest/gtest.h>
#include <filesystem>
#include <fstream>
#include <vector>
#include <string>
#include <kaitai/kaitaistream.h>
#include <fstream>
#include <string>
#include <vector>
#include "nix_nar.h"
static const std::vector<std::string> NarFiles = {
"empty.nar",
"dot.nar",
"dotdot.nar",
"executable-after-contents.nar",
"invalid-tag-instead-of-contents.nar",
"name-after-node.nar",
"nul-character.nar",
"slash.nar",
};
class NarParseTest : public ::testing::TestWithParam<std::string>
{};
TEST_P(NarParseTest, ParseSucceeds)
{
const auto nar_file = GetParam();
const char * nars_dir_env = std::getenv("NIX_NARS_DIR");
if (nars_dir_env == nullptr) {
FAIL() << "NIX_NARS_DIR environment variable not set.";
}
const std::filesystem::path nar_file_path = std::filesystem::path(nars_dir_env) / "dot.nar";
ASSERT_TRUE(std::filesystem::exists(nar_file_path)) << "Missing test file: " << nar_file_path;
std::ifstream ifs(nar_file_path, std::ifstream::binary);
ASSERT_TRUE(ifs.is_open()) << "Failed to open file: " << nar_file;
kaitai::kstream ks(&ifs);
nix_nar_t nar(&ks);
ASSERT_TRUE(nar.root_node() != nullptr) << "Failed to parse NAR file: " << nar_file;
}
INSTANTIATE_TEST_SUITE_P(AllNarFiles, NarParseTest, ::testing::ValuesIn(NarFiles));
+5 -13
View File
@@ -116,11 +116,7 @@ MixFlakeOptions::MixFlakeOptions()
.labels = {"input-path"},
.handler = {[&](std::string s) {
warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version.");
auto path = flake::NonEmptyInputAttrPath::parse(s);
if (!path)
throw UsageError(
"--update-input was passed a zero-length input path, which would refer to the flake itself, not an input");
lockFlags.inputUpdates.insert(*path);
lockFlags.inputUpdates.insert(flake::parseInputAttrPath(s));
}},
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
@@ -129,18 +125,14 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "override-input",
.description =
"Override a specific flake input (e.g. `dwarffs/nixpkgs`). The input path must not be empty. This implies `--no-write-lock-file`.",
.description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
.category = category,
.labels = {"input-path", "flake-url"},
.handler = {[&](std::string inputAttrPath, std::string flakeRef) {
lockFlags.writeLockFile = false;
auto path = flake::NonEmptyInputAttrPath::parse(inputAttrPath);
if (!path)
throw UsageError(
"--override-input was passed a zero-length input path, which would refer to the flake itself, not an input");
lockFlags.inputOverrides.insert_or_assign(
std::move(*path), parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true));
flake::parseInputAttrPath(inputAttrPath),
parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true));
}},
.completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
if (n == 0) {
@@ -157,7 +149,7 @@ MixFlakeOptions::MixFlakeOptions()
.category = category,
.labels = {"flake-lock-path"},
.handler = {[&](std::string lockFilePath) {
lockFlags.referenceLockFilePath = {makeFSSourceAccessor(absPath(lockFilePath)), CanonPath::root};
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
}},
.completer = completePath,
});
-14
View File
@@ -1,14 +0,0 @@
#include <benchmark/benchmark.h>
#include "nix/expr/eval-gc.hh"
#include "nix/store/globals.hh"
int main(int argc, char ** argv)
{
nix::initLibStore(false);
nix::initGC();
::benchmark::Initialize(&argc, argv);
::benchmark::RunSpecifiedBenchmarks();
return 0;
}
-55
View File
@@ -1,55 +0,0 @@
#include <benchmark/benchmark.h>
#include "nix/expr/eval.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/store/store-open.hh"
using namespace nix;
static std::string mkDynamicAttrsExpr(size_t attrCount)
{
std::string res;
res.reserve(attrCount * 24);
res += "{ ";
for (size_t i = 0; i < attrCount; ++i) {
res += "${\"a";
res += std::to_string(i);
res += "\"} = ";
res += std::to_string(i);
res += "; ";
}
res += "}";
return res;
}
static void BM_EvalDynamicAttrs(benchmark::State & state)
{
const auto attrCount = static_cast<size_t>(state.range(0));
const auto exprStr = mkDynamicAttrsExpr(attrCount);
for (auto _ : state) {
state.PauseTiming();
auto store = openStore("dummy://");
fetchers::Settings fetchSettings{};
bool readOnlyMode = true;
EvalSettings evalSettings{readOnlyMode};
evalSettings.nixPath = {};
EvalState st({}, store, fetchSettings, evalSettings, nullptr);
Expr * expr = st.parseExprFromString(exprStr, st.rootPath(CanonPath::root));
Value v;
state.ResumeTiming();
st.eval(expr, v);
st.forceValue(v, noPos);
benchmark::DoNotOptimize(v);
}
state.SetItemsProcessed(state.iterations() * attrCount);
}
BENCHMARK(BM_EvalDynamicAttrs)->Arg(100)->Arg(500)->Arg(2'000);
-64
View File
@@ -1,64 +0,0 @@
#include <benchmark/benchmark.h>
#include "nix/expr/get-drvs.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/store/store-open.hh"
#include "nix/util/fmt.hh"
using namespace nix;
namespace {
struct GetDerivationsEnv
{
ref<Store> store = openStore("dummy://");
fetchers::Settings fetchSettings{};
bool readOnlyMode = true;
EvalSettings evalSettings{readOnlyMode};
EvalState state;
Bindings * autoArgs = nullptr;
Value attrsValue;
explicit GetDerivationsEnv(size_t attrCount)
: evalSettings([&]() {
EvalSettings settings{readOnlyMode};
settings.nixPath = {};
return settings;
}())
, state({}, store, fetchSettings, evalSettings, nullptr)
{
autoArgs = state.buildBindings(0).finish();
auto attrs = state.buildBindings(attrCount);
for (size_t i = 0; i < attrCount; ++i) {
auto name = fmt("pkg%|1$06d|", i);
auto sym = state.symbols.create(name);
auto & v = attrs.alloc(sym);
v.mkInt(i);
}
attrsValue.mkAttrs(attrs.finish());
}
};
} // namespace
static void BM_GetDerivationsAttrScan(benchmark::State & state)
{
const auto attrCount = static_cast<size_t>(state.range(0));
GetDerivationsEnv env(attrCount);
for (auto _ : state) {
PackageInfos drvs;
getDerivations(
env.state, env.attrsValue, /*pathPrefix=*/"", *env.autoArgs, drvs, /*ignoreAssertionFailures=*/true);
benchmark::DoNotOptimize(drvs.size());
}
state.SetItemsProcessed(state.iterations() * attrCount);
}
BENCHMARK(BM_GetDerivationsAttrScan)->Arg(1'000)->Arg(5'000)->Arg(10'000);
-30
View File
@@ -87,33 +87,3 @@ test(
},
protocol : 'gtest',
)
# Build benchmarks if enabled
if get_option('benchmarks')
gbenchmark = dependency('benchmark', required : true)
benchmark_sources = files(
'bench-main.cc',
'dynamic-attrs-bench.cc',
'get-drvs-bench.cc',
'regex-cache-bench.cc',
)
benchmark_exe = executable(
'nix-expr-benchmarks',
benchmark_sources,
config_priv_h,
dependencies : deps_private_subproject + deps_private + deps_other + [
gbenchmark,
],
include_directories : include_dirs,
link_args : linker_export_flags,
install : true,
cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [],
)
benchmark(
'nix-expr-benchmarks',
benchmark_exe,
)
endif
-9
View File
@@ -1,9 +0,0 @@
# vim: filetype=meson
option(
'benchmarks',
type : 'boolean',
value : false,
description : 'Build benchmarks (requires gbenchmark)',
yield : true,
)
+1 -1
View File
@@ -33,7 +33,7 @@ mkMesonExecutable (finalAttrs: {
../../.version
./.version
./meson.build
./meson.options
# ./meson.options
(fileset.fileFilter (file: file.hasExt "cc") ./.)
(fileset.fileFilter (file: file.hasExt "hh") ./.)
];
-45
View File
@@ -1,45 +0,0 @@
#include <benchmark/benchmark.h>
#include "nix/expr/eval.hh"
#include "nix/expr/eval-settings.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/store/store-open.hh"
using namespace nix;
static void BM_EvalManyBuiltinsMatchSameRegex(benchmark::State & state)
{
static constexpr int iterations = 5'000;
static constexpr std::string_view exprStr =
"builtins.foldl' "
"(acc: _: acc + builtins.length (builtins.match \"a\" \"a\")) "
"0 "
"(builtins.genList (x: x) "
"5000)";
for (auto _ : state) {
state.PauseTiming();
auto store = openStore("dummy://");
fetchers::Settings fetchSettings{};
bool readOnlyMode = true;
EvalSettings evalSettings{readOnlyMode};
evalSettings.nixPath = {};
EvalState st({}, store, fetchSettings, evalSettings, nullptr);
Expr * expr = st.parseExprFromString(std::string(exprStr), st.rootPath(CanonPath::root));
Value v;
state.ResumeTiming();
st.eval(expr, v);
st.forceValue(v, noPos);
benchmark::DoNotOptimize(v);
}
state.SetItemsProcessed(state.iterations() * iterations);
}
BENCHMARK(BM_EvalManyBuiltinsMatchSameRegex);
+9 -9
View File
@@ -1,6 +1,5 @@
#include "nix/expr/attr-path.hh"
#include "nix/expr/eval-inline.hh"
#include "nix/util/util.hh"
#include "nix/util/strings-inline.hh"
namespace nix {
@@ -138,15 +137,16 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
auto fail = [fn]() { throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); };
auto colon = fn.rfind(':');
if (colon == std::string::npos)
try {
auto colon = fn.rfind(':');
if (colon == std::string::npos)
fail();
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) {
fail();
auto lineno = string2Int<uint32_t>(std::string_view(fn).substr(colon + 1));
if (!lineno)
fail();
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, *lineno};
unreachable();
}
}
} // namespace nix
+1 -1
View File
@@ -265,7 +265,7 @@ struct AttrDb
case AttrType::String: {
NixStringContext context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), " "))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
context.insert(NixStringContextElem::parse(s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
-1
View File
@@ -111,7 +111,6 @@ template class EvalErrorBuilder<TypeError>;
template class EvalErrorBuilder<UndefinedVarError>;
template class EvalErrorBuilder<MissingArgumentError>;
template class EvalErrorBuilder<InfiniteRecursionError>;
template class EvalErrorBuilder<StackOverflowError>;
template class EvalErrorBuilder<InvalidPathError>;
template class EvalErrorBuilder<IFDError>;
-6
View File
@@ -2408,8 +2408,6 @@ BackedStringView EvalState::coerceToString(
bool copyToStore,
bool canonicalizePath)
{
auto _level = addCallDepth(pos);
forceValue(v, pos);
if (v.type() == nString) {
@@ -2629,8 +2627,6 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
// `assert a == b; x` are critical for our users' testing UX.
void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{
auto _level = addCallDepth(pos);
// This implementation must match eqValues.
forceValue(v1, pos);
forceValue(v2, pos);
@@ -2837,8 +2833,6 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
// This implementation must match assertEqValues
bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{
auto _level = addCallDepth(pos);
forceValue(v1, pos);
forceValue(v2, pos);
+2 -25
View File
@@ -213,8 +213,6 @@ StringSet PackageInfo::queryMetaNames()
bool PackageInfo::checkMeta(Value & v)
{
auto _level = state->addCallDepth(v.determinePos(noPos));
state->forceValue(v, v.determinePos(noPos));
if (v.type() == nList) {
for (auto elem : v.listView())
@@ -369,26 +367,7 @@ static std::string addToPath(const std::string & s1, std::string_view s2)
return s1.empty() ? std::string(s2) : s1 + "." + s2;
}
static bool isAttrPathComponent(std::string_view symbol)
{
if (symbol.empty())
return false;
/* [A-Za-z_] */
unsigned char first = symbol[0];
if (!((first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z') || first == '_'))
return false;
/* [A-Za-z0-9-_+]* */
for (unsigned char c : symbol.substr(1)) {
if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_'
|| c == '+')
continue;
return false;
}
return true;
}
static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
static void getDerivations(
EvalState & state,
@@ -399,8 +378,6 @@ static void getDerivations(
Done & done,
bool ignoreAssertionFailures)
{
auto _level = state.addCallDepth(vIn.determinePos(noPos));
Value v;
state.autoCallFunction(autoArgs, vIn, v);
@@ -423,7 +400,7 @@ static void getDerivations(
std::string_view symbol{state.symbols[i->name]};
try {
debug("evaluating attribute '%1%'", symbol);
if (!isAttrPathComponent(symbol))
if (!std::regex_match(symbol.begin(), symbol.end(), attrRegex))
continue;
std::string pathPrefix2 = addToPath(pathPrefix, symbol);
if (combineChannels)
@@ -54,20 +54,6 @@ MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, EvalError);
MakeError(MissingArgumentError, EvalError);
MakeError(InfiniteRecursionError, EvalError);
/**
* Resource exhaustion error when evaluation exceeds max-call-depth.
* Inherits from EvalBaseError (not EvalError) because resource exhaustion
* should not be cached.
*/
struct StackOverflowError : public EvalBaseError
{
StackOverflowError(EvalState & state)
: EvalBaseError(state, "stack overflow; max-call-depth exceeded")
{
}
};
MakeError(IFDError, EvalBaseError);
struct InvalidPathError : public EvalError
+1 -1
View File
@@ -139,7 +139,7 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
inline CallDepth EvalState::addCallDepth(const PosIdx pos)
{
if (callDepth > settings.maxCallDepth)
error<StackOverflowError>().atPos(pos).debugThrow();
error<EvalBaseError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
return CallDepth(callDepth);
};
@@ -5,8 +5,6 @@
namespace nix {
class EvalState;
/**
* Print a value in the deprecated format used by `nix-instantiate --eval` and
* `nix-env` (for manifests).
@@ -17,6 +15,7 @@ class EvalState;
*
* See: https://github.com/NixOS/nix/issues/9730
*/
void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set<const void *> * seen, size_t depth = 0);
void printAmbiguous(
Value & v, const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen, int depth);
} // namespace nix
+18 -28
View File
@@ -3002,19 +3002,16 @@ static RegisterPrimOp primop_path({
- recursive\
When `false`, when `path` is added to the store it is with a
[flat hash](@docroot@/store/file-system-object/content-address.md#serial-flat),
rather than a hash of the
[NAR serialization](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
of the file. Thus, `path` must refer to a regular file, not a
flat hash, rather than a hash of the NAR serialization of the
file. Thus, `path` must refer to a regular file, not a
directory. This allows similar behavior to `fetchurl`. Defaults
to `true`.
- sha256\
When provided, this is the expected
[content hash](@docroot@/store/file-system-object/content-address.md)
of the path. Evaluation fails if the hash is incorrect,
and providing a hash allows `builtins.path` to be used even
when the `pure-eval` nix config option is on.
When provided, this is the expected hash of the file at the
path. Evaluation fails if the hash is incorrect, and
providing a hash allows `builtins.path` to be used even when the
`pure-eval` nix config option is on.
)",
.fun = prim_path,
});
@@ -4712,28 +4709,21 @@ static RegisterPrimOp primop_convertHash({
struct RegexCache
{
struct Entry
boost::concurrent_flat_map<std::string, std::regex, StringViewHash, std::equal_to<>> cache;
std::regex get(std::string_view re)
{
ref<const std::regex> regex;
Entry(const char * s, size_t count)
: regex(make_ref<const std::regex>(s, count, std::regex::extended))
{
}
};
boost::concurrent_flat_map<std::string, Entry, StringViewHash, std::equal_to<>> cache;
ref<const std::regex> get(std::string_view re)
{
std::optional<ref<const std::regex>> regex;
std::regex regex;
/* No std::regex constructor overload from std::string_view, but can be constructed
from a pointer + size or an iterator range. */
cache.try_emplace_and_cvisit(
re,
/*s=*/re.data(),
/*count=*/re.size(),
[&regex](const auto & kv) { regex = kv.second.regex; },
[&regex](const auto & kv) { regex = kv.second.regex; });
return *regex;
std::regex::extended,
[&regex](const auto & kv) { regex = kv.second; },
[&regex](const auto & kv) { regex = kv.second; });
return regex;
}
};
@@ -4755,7 +4745,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v)
state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match");
std::cmatch match;
if (!std::regex_match(str.begin(), str.end(), match, *regex)) {
if (!std::regex_match(str.begin(), str.end(), match, regex)) {
v.mkNull();
return;
}
@@ -4828,7 +4818,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value ** args, Value & v)
const auto str =
state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split");
auto begin = std::cregex_iterator(str.begin(), str.end(), *regex);
auto begin = std::cregex_iterator(str.begin(), str.end(), regex);
auto end = std::cregex_iterator();
// Any matches results are surrounded by non-matching results.
+4 -4
View File
@@ -243,7 +243,7 @@ static RegisterPrimOp primop_fetchClosure({
```nix
builtins.fetchClosure {
fromStore = "https://cache.nixos.org";
fromPath = /nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1;
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
}
```
@@ -258,8 +258,8 @@ static RegisterPrimOp primop_fetchClosure({
use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
```console
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1
rewrote '/nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
```
Alternatively, set `toPath = ""` and find the correct `toPath` in the error message.
@@ -271,7 +271,7 @@ static RegisterPrimOp primop_fetchClosure({
```nix
builtins.fetchClosure {
fromStore = "https://cache.nixos.org";
fromPath = /nix/store/nph9br6y2dmciy6q3dj3fwk2brdlr4gh-git-2.33.1;
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
inputAddressed = true;
}
```
+10 -8
View File
@@ -2,17 +2,19 @@
#include "nix/expr/print.hh"
#include "nix/util/signals.hh"
#include "nix/expr/eval.hh"
#include "nix/expr/eval-error.hh"
namespace nix {
// See: https://github.com/NixOS/nix/issues/9730
void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set<const void *> * seen, size_t depth)
void printAmbiguous(
Value & v, const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen, int depth)
{
checkInterrupt();
if (depth > state.settings.maxCallDepth)
state.error<StackOverflowError>().atPos(v.determinePos(noPos)).debugThrow();
if (depth <= 0) {
str << "«too deep»";
return;
}
switch (v.type()) {
case nInt:
str << v.integer();
@@ -34,9 +36,9 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set<c
str << "«repeated»";
else {
str << "{ ";
for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
str << state.symbols[i->name] << " = ";
printAmbiguous(state, *i->value, str, seen, depth + 1);
for (auto & i : v.attrs()->lexicographicOrder(symbols)) {
str << symbols[i->name] << " = ";
printAmbiguous(*i->value, symbols, str, seen, depth - 1);
str << "; ";
}
str << "}";
@@ -52,7 +54,7 @@ void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set<c
str << "[ ";
for (auto v2 : v.listView()) {
if (v2)
printAmbiguous(state, *v2, str, seen, depth + 1);
printAmbiguous(*v2, symbols, str, seen, depth - 1);
else
str << "(nullptr)";
str << " ";
-15
View File
@@ -537,16 +537,6 @@ private:
output.flush();
checkInterrupt();
// Catch infinite recursion before it overflows the C++ stack.
// Non-cyclic structures can be infinitely deep when values are
// lazily produced (e.g., `let f = n: { inner = f (n + 1); }; in f 0`).
// We check print depth against max-call-depth rather than incrementing
// the callDepth counter, because accessing an attribute is not a call.
// Other places do increment callDepth for simplicity, but that is
// technically incorrect.
if (depth > state.settings.maxCallDepth)
state.error<StackOverflowError>().atPos(v.determinePos(noPos)).debugThrow();
try {
if (options.force) {
state.forceValue(v, v.determinePos(noPos));
@@ -602,11 +592,6 @@ private:
printUnknown();
break;
}
} catch (StackOverflowError &) {
// Always re-throw because stack overflow is a serious condition
// that expressions should avoid, unlike say `throw`, which can
// be part of legitimate expression patterns.
throw;
} catch (Error & e) {
if (options.errors == ErrorPrintBehavior::Throw
|| (options.errors == ErrorPrintBehavior::ThrowTopLevel && depth == 0)) {
-2
View File
@@ -66,8 +66,6 @@ static void printValueAsXML(
{
checkInterrupt();
auto _level = state.addCallDepth(pos);
if (strict)
state.forceValue(v, pos);
+1 -2
View File
@@ -115,10 +115,9 @@ TEST_F(GitUtilsTest, sink_hardlink)
try {
sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello"));
sink->flush();
FAIL() << "Expected an exception";
} catch (const nix::Error & e) {
ASSERT_THAT(e.msg(), testing::HasSubstr("does not exist"));
ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target"));
ASSERT_THAT(e.msg(), testing::HasSubstr("/hello"));
ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link"));
}
+212 -230
View File
@@ -369,26 +369,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{
// TODO: as an optimization, it would be nice to include `this` in the pool.
return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
auto repo = make_ref<GitRepoImpl>(path, options);
/* Monkey-patching the pack backend to only read the pack directory
once. Otherwise it will do a readdir for each added oid when it's
not found and that translates to ~6 syscalls. Since we are never
writing pack files until flushing we can force the odb backend to
read the directory just once. It's very convenient that the vtable is
semi-public interface and is up for grabs.
This is purely an optimization for our use-case with a tarball cache.
libgit2 calls refresh() if the backend provides it when an oid isn't found.
We are only writing objects to a mempack (it has higher priority) and there isn't
a realistic use-case where a previously missing object would appear from thin air
on the disk (unless another process happens to be unpacking a similar tarball to
the cache at the same time, but that's a very unrealistic scenario).
*/
if (auto * backend = repo->packBackend)
backend->refresh = nullptr;
return repo;
return make_ref<GitRepoImpl>(path, options);
});
}
@@ -687,7 +668,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
keyDecoded = base64::decode(k.key);
} catch (Error & e) {
e.addTrace({}, "while decoding public key '%s' used for git signature", k.key);
throw;
}
auto fingerprint =
trim(hashString(HashAlgorithm::SHA256, keyDecoded).to_string(nix::HashFormat::Base64, false), "=");
@@ -1078,155 +1058,185 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
ref<GitRepoImpl> repo;
Pool<GitRepoImpl> repoPool;
struct PendingDir
{
std::string name;
TreeBuilder builder;
};
unsigned int concurrency = std::min(std::thread::hardware_concurrency(), 10U);
std::vector<PendingDir> pendingDirs;
ThreadPool workers{concurrency};
/**
* Temporary buffer used by createRegularFile for storing small file contents.
*/
std::string regularFileContentsBuffer;
/** Total file contents in flight. */
std::atomic<size_t> totalBufSize{0};
/**
* If repo has a non-null packBackend, this has a copy of the refresh function
* from the backend virtual table. This is needed to restore it after we've flushed
* the sink. We modify it to avoid unnecessary I/O on non-existent oids.
*/
decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
static constexpr std::size_t maxBufSize = 16 * 1024 * 1024;
void pushBuilder(std::string name)
{
const git_tree_entry * entry;
Tree prevTree = nullptr;
if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
/* Clone a tree that we've already finished. This happens
if a tarball has directory entries that are not
contiguous. */
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
throw Error("parent of '%s' is not a directory", name);
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
}
git_treebuilder * b;
if (git_treebuilder_new(&b, *repo, prevTree.get()))
throw Error("creating a tree builder: %s", git_error_last()->message);
pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
};
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
: repo(repo)
, repoPool(repo->getPool())
{
/* Monkey-patching the pack backend to only read the pack directory
once. Otherwise it will do a readdir for each added oid when it's
not found and that translates to ~6 syscalls. Since we are never
writing pack files until flushing we can force the odb backend to
read the directory just once. It's very convenient that the vtable is
semi-public interface and is up for grabs.
This is purely an optimization for our use-case with a tarball cache.
libgit2 calls refresh() if the backend provides it when an oid isn't found.
We are only writing objects to a mempack (it has higher priority) and there isn't
a realistic use-case where a previously missing object would appear from thin air
on the disk (unless another process happens to be unpacking a similar tarball to
the cache at the same time, but that's a very unrealistic scenario).
*/
if (auto * backend = repo->packBackend) {
if (backend->refresh(backend)) /* Refresh just once manually. */
throw Error("refreshing packfiles: %s", git_error_last()->message);
/* Save the function pointer to restore it later in flush() and
unset it in the vtable. libgit2 does nothing if it's a nullptr:
https://github.com/libgit2/libgit2/blob/58d9363f02f1fa39e46d49b604f27008e75b72f2/src/libgit2/odb.c#L1922
*/
packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
}
pushBuilder("");
}
~GitFileSystemObjectSinkImpl()
std::pair<git_oid, std::string> popBuilder()
{
// Make sure the worker threads are destroyed before any state
// they're referring to.
workers.shutdown();
}
struct Child;
/// A directory to be written as a Git tree.
struct Directory
{
std::map<std::string, Child> children;
std::optional<git_oid> oid;
Child & lookup(const CanonPath & path)
{
assert(!path.isRoot());
auto parent = path.parent();
auto cur = this;
for (auto & name : *parent) {
auto i = cur->children.find(std::string(name));
if (i == cur->children.end())
throw Error("path '%s' does not exist", path);
auto dir = std::get_if<Directory>(&i->second.file);
if (!dir)
throw Error("path '%s' has a non-directory parent", path);
cur = dir;
}
auto i = cur->children.find(std::string(*path.baseName()));
if (i == cur->children.end())
throw Error("path '%s' does not exist", path);
return i->second;
}
assert(!pendingDirs.empty());
auto pending = std::move(pendingDirs.back());
git_oid oid;
if (git_treebuilder_write(&oid, pending.builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
pendingDirs.pop_back();
return {oid, pending.name};
};
size_t nextId = 0; // for Child.id
struct Child
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
{
git_filemode_t mode;
std::variant<Directory, git_oid> file;
/// Sequential numbering of the file in the tarball. This is
/// used to make sure we only import the latest version of a
/// path.
size_t id{0};
assert(!pendingDirs.empty());
auto & pending = pendingDirs.back();
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
};
struct State
void updateBuilders(std::span<const std::string> names)
{
Directory root;
};
// Find the common prefix of pendingDirs and names.
size_t prefixLen = 0;
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
break;
Sync<State> _state;
void addNode(State & state, const CanonPath & path, Child && child)
{
assert(!path.isRoot());
auto parent = path.parent();
Directory * cur = &state.root;
for (auto & i : *parent) {
auto child = std::get_if<Directory>(
&cur->children.emplace(std::string(i), Child{GIT_FILEMODE_TREE, {Directory()}}).first->second.file);
assert(child);
cur = child;
// Finish the builders that are not part of the common prefix.
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
auto [oid, name] = popBuilder();
addToTree(name, oid, GIT_FILEMODE_TREE);
}
std::string name(*path.baseName());
// Create builders for the new directories.
for (auto n = prefixLen; n < names.size(); ++n)
pushBuilder(names[n]);
};
if (auto prev = cur->children.find(name); prev == cur->children.end() || prev->second.id < child.id)
cur->children.insert_or_assign(name, std::move(child));
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
{
std::span<const std::string> pathComponents2{pathComponents};
updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
return true;
}
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
{
checkInterrupt();
/* Multithreaded blob writing. We read the incoming file data into memory and asynchronously write it to a Git
blob object. However, to avoid unbounded memory usage, if the amount of data in flight exceeds a threshold,
we switch to writing directly to a Git write stream. */
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
if (!prepareDirs(pathComponents, false))
return;
using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
if (stream)
stream->free(stream);
})>;
/* Maximum file size that gets buffered in memory before flushing to a WriteStream,
that's backed by a temporary objects/streamed_git2_* file. We should avoid that
for common cases, since creating (and deleting) a temporary file for each blob
is insanely expensive. */
static constexpr std::size_t maxBufferSize = 1024 * 1024; /* 1 MiB */
struct CRF : CreateRegularFileSink
{
CanonPath path;
GitFileSystemObjectSinkImpl & parent;
const CanonPath & path;
GitFileSystemObjectSinkImpl & back;
WriteStream stream;
std::optional<decltype(parent.repoPool)::Handle> repo;
std::string contents;
std::string & contents;
bool executable = false;
CRF(CanonPath path, GitFileSystemObjectSinkImpl & parent)
: path(std::move(path))
, parent(parent)
CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, std::string & regularFileContentsBuffer)
: path(path)
, back(back)
, stream(nullptr)
, contents(regularFileContentsBuffer)
{
contents.clear();
}
~CRF()
void writeToStream(std::string_view data)
{
parent.totalBufSize -= contents.size();
/* Lazily create the stream. */
if (!stream) {
::git_writestream * stream2 = nullptr;
if (git_blob_create_from_stream(&stream2, *back.repo, nullptr))
throw Error("creating a blob stream object: %s", git_error_last()->message);
stream = WriteStream{stream2};
assert(stream);
}
if (stream->write(stream.get(), data.data(), data.size()))
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
}
void operator()(std::string_view data) override
{
if (!stream) {
contents.append(data);
parent.totalBufSize += data.size();
/* Already in slow path. Just write to the slow stream. */
if (stream) {
writeToStream(data);
return;
}
if (parent.totalBufSize > parent.maxBufSize) {
repo.emplace(parent.repoPool.get());
if (git_blob_create_from_stream(Setter(stream), **repo, nullptr))
throw Error("creating a blob stream object: %s", git_error_last()->message);
if (stream->write(stream.get(), contents.data(), contents.size()))
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
parent.totalBufSize -= contents.size();
contents.clear();
}
} else {
if (stream->write(stream.get(), data.data(), data.size()))
throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message);
contents += data;
if (contents.size() > maxBufferSize) {
writeToStream(contents); /* Will initialize stream. */
contents.clear();
}
}
@@ -1234,140 +1244,112 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
executable = true;
}
};
} crf{path, *this, regularFileContentsBuffer};
auto crf = std::make_shared<CRF>(path, *this);
func(crf);
func(*crf);
auto id = nextId++;
if (crf->stream) {
/* Finish the slow path by creating the blob object synchronously.
Call .release(), since git_blob_create_from_stream_commit
git_oid oid;
if (crf.stream) {
/* Call .release(), since git_blob_create_from_stream_commit
acquires ownership and frees the stream. */
git_oid oid;
if (git_blob_create_from_stream_commit(&oid, crf->stream.release()))
if (git_blob_create_from_stream_commit(&oid, crf.stream.release()))
throw Error("creating a blob object for '%s': %s", path, git_error_last()->message);
addNode(
*_state.lock(),
crf->path,
Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id});
return;
} else {
if (git_blob_create_from_buffer(&oid, *repo, crf.contents.data(), crf.contents.size()))
throw Error(
"creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
}
/* Fast path: create the blob object in a separate thread. */
workers.enqueue([this, crf{std::move(crf)}, id]() {
auto repo(repoPool.get());
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, crf->contents.data(), crf->contents.size()))
throw Error(
"creating a blob object for '%s' from in-memory buffer: %s", crf->path, git_error_last()->message);
addNode(
*_state.lock(),
crf->path,
Child{crf->executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id});
});
addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
}
void createDirectory(const CanonPath & path) override
{
if (path.isRoot())
return;
auto state(_state.lock());
addNode(*state, path, {GIT_FILEMODE_TREE, Directory()});
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
(void) prepareDirs(pathComponents, true);
}
void createSymlink(const CanonPath & path, const std::string & target) override
{
workers.enqueue([this, path, target]() {
auto repo(repoPool.get());
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
if (!prepareDirs(pathComponents, false))
return;
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
throw Error(
"creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
auto state(_state.lock());
addNode(*state, path, Child{GIT_FILEMODE_LINK, oid});
});
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
}
std::map<CanonPath, CanonPath> hardLinks;
void createHardlink(const CanonPath & path, const CanonPath & target) override
{
hardLinks.insert_or_assign(path, target);
std::vector<std::string> pathComponents;
for (auto & c : path)
pathComponents.emplace_back(c);
if (!prepareDirs(pathComponents, false))
return;
// We can't just look up the path from the start of the root, since
// some parent directories may not have finished yet, so we compute
// a relative path that helps us find the right git_tree_builder or object.
auto relTarget = CanonPath(path).parent()->makeRelative(target);
auto dir = pendingDirs.rbegin();
// For each ../ component at the start, go up one directory.
// CanonPath::makeRelative() always puts all .. elements at the start,
// so they're all handled by this loop:
std::string_view relTargetLeft(relTarget);
while (hasPrefix(relTargetLeft, "../")) {
if (dir == pendingDirs.rend())
throw Error("invalid hard link target '%s' for path '%s'", target, path);
++dir;
relTargetLeft = relTargetLeft.substr(3);
}
if (dir == pendingDirs.rend())
throw Error("invalid hard link target '%s' for path '%s'", target, path);
// Look up the remainder of the target, starting at the
// top-most `git_treebuilder`.
std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
Object tree; // needed to keep `entry` alive
const git_tree_entry * entry = nullptr;
for (auto & c : CanonPath(relTargetLeft)) {
if (auto builder = std::get_if<git_treebuilder *>(&curDir)) {
assert(*builder);
if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str())))
throw Error("cannot find hard link target '%s' for path '%s'", target, path);
curDir = *git_tree_entry_id(entry);
} else if (auto oid = std::get_if<git_oid>(&curDir)) {
tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE);
if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str())))
throw Error("cannot find hard link target '%s' for path '%s'", target, path);
curDir = *git_tree_entry_id(entry);
}
}
assert(entry);
addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
}
Hash flush() override
{
workers.process();
updateBuilders({});
/* Create hard links. */
{
auto state(_state.lock());
for (auto & [path, target] : hardLinks) {
if (target.isRoot())
continue;
try {
auto child = state->root.lookup(target);
auto oid = std::get_if<git_oid>(&child.file);
if (!oid)
throw Error("cannot create a hard link to a directory");
addNode(*state, path, {child.mode, *oid});
} catch (Error & e) {
e.addTrace(nullptr, "while creating a hard link from '%s' to '%s'", path, target);
throw;
}
}
auto [oid, _name] = popBuilder();
if (auto * backend = repo->packBackend) {
/* We are done writing blobs, can restore refresh functionality. */
backend->refresh = packfileOdbRefresh;
}
// Flush all repo objects to disk.
{
auto repos = repoPool.clear();
ThreadPool workers{repos.size()};
for (auto & repo : repos)
workers.enqueue([repo]() { repo->flush(); });
workers.process();
}
// Write the Git trees to disk. Would be nice to have this multithreaded too, but that's hard because a tree
// can't refer to an object that hasn't been written yet. Also it doesn't make a big difference for performance.
auto repo(repoPool.get());
[&](this const auto & visit, Directory & node) -> void {
checkInterrupt();
// Write the child directories.
for (auto & child : node.children)
if (auto dir = std::get_if<Directory>(&child.second.file))
visit(*dir);
// Write this directory.
git_treebuilder * b;
if (git_treebuilder_new(&b, *repo, nullptr))
throw Error("creating a tree builder: %s", git_error_last()->message);
TreeBuilder builder(b);
for (auto & [name, child] : node.children) {
auto oid_p = std::get_if<git_oid>(&child.file);
auto oid = oid_p ? *oid_p : std::get<Directory>(child.file).oid.value();
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
}
git_oid oid;
if (git_treebuilder_write(&oid, builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
node.oid = oid;
}(_state.lock()->root);
repo->flush();
return toHash(_state.lock()->root.oid.value());
return toHash(oid);
}
};
+10 -8
View File
@@ -213,24 +213,26 @@ struct MercurialInputScheme : InputScheme
runHg({"status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0"}),
"\0"s);
auto accessor = makeFSSourceAccessor(absPath(actualUrl));
std::filesystem::path actualPath(absPath(actualUrl));
PathFilter filter = [&](const Path & p) -> bool {
auto cp = CanonPath(p);
auto st = accessor->lstat(cp);
assert(hasPrefix(p, actualPath.string()));
std::string file(p, actualPath.string().size() + 1);
if (st.type == SourceAccessor::tDirectory) {
auto prefix = cp.rel() + "/";
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(cp.rel());
return files.count(file);
};
auto storePath = store.addToStore(
input.getName(),
{accessor, CanonPath::root},
{getFSSourceAccessor(), CanonPath(actualPath.string())},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256,
{},
@@ -332,7 +334,7 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir / ".hg_archival.txt");
auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root});
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())});
Attrs infoAttrs({
{"revCount", (uint64_t) revCount},
+1
View File
@@ -165,6 +165,7 @@ struct PathInputScheme : InputScheme
// To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache.
auto info = store.queryPathInfo(*storePath);
accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert(
+2 -5
View File
@@ -162,11 +162,8 @@ nix_err nix_flake_lock_flags_add_input_override(
{
nix_clear_err(context);
try {
auto path = nix::flake::NonEmptyInputAttrPath::parse(inputPath);
if (!path)
throw nix::UsageError(
"input override path cannot be zero-length; it would refer to the flake itself, not an input");
flags->lockFlags->inputOverrides.emplace(std::move(*path), *flakeRef->flakeRef);
auto path = nix::flake::parseInputAttrPath(inputPath);
flags->lockFlags->inputOverrides.emplace(path, *flakeRef->flakeRef);
if (flags->lockFlags->writeLockFile) {
return nix_flake_lock_flags_set_mode_virtual(context, flags);
}
+1 -2
View File
@@ -160,9 +160,8 @@ nix_err nix_flake_lock_flags_set_mode_write_as_needed(nix_c_context * context, n
* @brief Add input overrides to the lock flags
* @param[out] context Optional, stores error information
* @param[in] flags The flags to modify
* @param[in] inputPath The input path to override (must not be empty)
* @param[in] inputPath The input path to override
* @param[in] flakeRef The flake reference to use as the override
* @return NIX_ERR_NIX_ERROR if inputPath is empty
*
* This switches the `flags` to `nix_flake_lock_flags_set_mode_virtual` if not in mode
* `nix_flake_lock_flags_set_mode_check`.
-58
View File
@@ -384,62 +384,4 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
nix_flake_settings_free(settings);
}
TEST_F(nix_api_store_test, nix_api_flake_lock_flags_add_input_override_empty_path)
{
auto tmpDir = nix::createTempDir();
nix::AutoDelete delTmpDir(tmpDir, true);
nix::writeFile(tmpDir / "flake.nix", R"(
{
outputs = { ... }: { };
}
)");
nix_libstore_init(ctx);
assert_ctx_ok();
auto fetchSettings = nix_fetchers_settings_new(ctx);
assert_ctx_ok();
ASSERT_NE(nullptr, fetchSettings);
auto settings = nix_flake_settings_new(ctx);
assert_ctx_ok();
ASSERT_NE(nullptr, settings);
auto lockFlags = nix_flake_lock_flags_new(ctx, settings);
assert_ctx_ok();
ASSERT_NE(nullptr, lockFlags);
auto parseFlags = nix_flake_reference_parse_flags_new(ctx, settings);
assert_ctx_ok();
ASSERT_NE(nullptr, parseFlags);
auto r0 = nix_flake_reference_parse_flags_set_base_directory(
ctx, parseFlags, tmpDir.string().c_str(), tmpDir.string().size());
assert_ctx_ok();
ASSERT_EQ(NIX_OK, r0);
nix_flake_reference * flakeReference = nullptr;
std::string fragment;
nix_flake_reference_and_fragment_from_string(
ctx, fetchSettings, settings, parseFlags, ".", 1, &flakeReference, OBSERVE_STRING(fragment));
assert_ctx_ok();
ASSERT_NE(nullptr, flakeReference);
// Test that empty input path is rejected (issue #14816)
auto r = nix_flake_lock_flags_add_input_override(ctx, lockFlags, "", flakeReference);
ASSERT_EQ(NIX_ERR_NIX_ERROR, r);
assert_ctx_err();
// Verify error message contains expected text
const char * errMsg = nix_err_msg(nullptr, ctx, nullptr);
ASSERT_NE(nullptr, errMsg);
ASSERT_NE(std::string(errMsg).find("input override path cannot be zero-length"), std::string::npos);
nix_flake_reference_free(flakeReference);
nix_flake_reference_parse_flags_free(parseFlags);
nix_flake_lock_flags_free(lockFlags);
nix_flake_settings_free(settings);
}
} // namespace nixC
+20 -19
View File
@@ -451,10 +451,9 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
std::optional<InputAttrPath> parentInputAttrPath; // FIXME: rename to inputAttrPathPrefix?
};
std::map<NonEmptyInputAttrPath, OverrideTarget> overrides;
std::set<NonEmptyInputAttrPath> explicitCliOverrides;
std::set<NonEmptyInputAttrPath> overridesUsed;
std::set<InputAttrPath> updatesUsed;
std::map<InputAttrPath, OverrideTarget> overrides;
std::set<InputAttrPath> explicitCliOverrides;
std::set<InputAttrPath> overridesUsed, updatesUsed;
std::map<ref<Node>, SourcePath> nodePaths;
for (auto & i : lockFlags.inputOverrides) {
@@ -511,7 +510,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
auto addOverrides =
[&](this const auto & addOverrides, const FlakeInput & input, const InputAttrPath & prefix) -> void {
for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputAttrPath = NonEmptyInputAttrPath::append(prefix, idOverride);
auto inputAttrPath(prefix);
inputAttrPath.push_back(idOverride);
if (inputOverride.ref || inputOverride.follows)
overrides.emplace(
inputAttrPath,
@@ -532,8 +532,9 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
/* Check whether this input has overrides for a
non-existent input. */
for (auto [inputAttrPath, inputOverride] : overrides) {
auto follow = inputAttrPath.inputName();
auto inputAttrPath2 = inputAttrPath.parent();
auto inputAttrPath2(inputAttrPath);
auto follow = inputAttrPath2.back();
inputAttrPath2.pop_back();
if (inputAttrPath2 == inputAttrPathPrefix && !flakeInputs.count(follow))
warn(
"input '%s' has an override for a non-existent input '%s'",
@@ -545,8 +546,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
necessary (i.e. if they're new or the flakeref changed
from what's in the lock file). */
for (auto & [id, input2] : flakeInputs) {
auto nonEmptyInputAttrPath = NonEmptyInputAttrPath::append(inputAttrPathPrefix, id);
auto inputAttrPath = nonEmptyInputAttrPath.get();
auto inputAttrPath(inputAttrPathPrefix);
inputAttrPath.push_back(id);
auto inputAttrPathS = printInputAttrPath(inputAttrPath);
debug("computing input '%s'", inputAttrPathS);
@@ -554,11 +555,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
/* Do we have an override for this input from one of the
ancestors? */
auto i = overrides.find(nonEmptyInputAttrPath);
auto i = overrides.find(inputAttrPath);
bool hasOverride = i != overrides.end();
bool hasCliOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath);
bool hasCliOverride = explicitCliOverrides.contains(inputAttrPath);
if (hasOverride)
overridesUsed.insert(nonEmptyInputAttrPath);
overridesUsed.insert(inputAttrPath);
auto input = hasOverride ? i->second.input : input2;
/* Resolve relative 'path:' inputs relative to
@@ -617,7 +618,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
updatesUsed.insert(inputAttrPath);
if (oldNode && !lockFlags.inputUpdates.count(nonEmptyInputAttrPath))
if (oldNode && !lockFlags.inputUpdates.count(inputAttrPath))
if (auto oldLock2 = get(oldNode->inputs, id))
if (auto oldLock3 = std::get_if<0>(&*oldLock2))
oldLock = *oldLock3;
@@ -636,10 +637,10 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
/* If we have this input in updateInputs, then we
must fetch the flake to update it. */
auto lb = lockFlags.inputUpdates.lower_bound(nonEmptyInputAttrPath);
auto lb = lockFlags.inputUpdates.lower_bound(inputAttrPath);
auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->get().size() > inputAttrPath.size()
&& std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->get().begin());
auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputAttrPath.size()
&& std::equal(inputAttrPath.begin(), inputAttrPath.end(), lb->begin());
FlakeInputs fakeInputs;
@@ -661,8 +662,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
// It is possible that the flake has changed,
// so we must confirm all the follows that are in the lock file are also in the
// flake.
auto overridePath =
NonEmptyInputAttrPath::append(nonEmptyInputAttrPath, i.first);
auto overridePath(inputAttrPath);
overridePath.push_back(i.first);
auto o = overrides.find(overridePath);
// If the override disappeared, we have to refetch the flake,
// since some of the inputs may not be present in the lock file.
@@ -716,7 +717,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
nuked the next time we update the lock
file. That is, overrides are sticky unless you
use --no-write-lock-file. */
auto inputIsOverride = explicitCliOverrides.contains(nonEmptyInputAttrPath);
auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath);
auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref;
if (input.isFlake) {
+2 -2
View File
@@ -205,13 +205,13 @@ struct LockFlags
/**
* Flake inputs to be overridden.
*/
std::map<NonEmptyInputAttrPath, FlakeRef> inputOverrides;
std::map<InputAttrPath, FlakeRef> inputOverrides;
/**
* Flake inputs to be updated. This means that any existing lock
* for those inputs will be ignored.
*/
std::set<NonEmptyInputAttrPath> inputUpdates;
std::set<InputAttrPath> inputUpdates;
};
LockedFlake
@@ -14,80 +14,6 @@ namespace nix::flake {
typedef std::vector<FlakeId> InputAttrPath;
/**
* A non-empty input attribute path.
*
* Input attribute paths identify inputs in a flake. An empty path would
* refer to the flake itself rather than an input, which contradicts the
* purpose of operations like override or update.
*/
class NonEmptyInputAttrPath
{
InputAttrPath path;
explicit NonEmptyInputAttrPath(InputAttrPath && p)
: path(std::move(p))
{
assert(!path.empty());
}
public:
/**
* Parse and validate a non-empty input attribute path.
* Returns std::nullopt if the path is empty.
*/
static std::optional<NonEmptyInputAttrPath> parse(std::string_view s);
/**
* Construct from an already-parsed path.
* Returns std::nullopt if the path is empty.
*/
static std::optional<NonEmptyInputAttrPath> make(InputAttrPath path);
/**
* Append an element to a path, creating a non-empty path.
* This is always safe because adding an element guarantees non-emptiness.
*/
static NonEmptyInputAttrPath append(const InputAttrPath & prefix, const FlakeId & element)
{
InputAttrPath path = prefix;
path.push_back(element);
return NonEmptyInputAttrPath{std::move(path)};
}
const InputAttrPath & get() const
{
return path;
}
operator const InputAttrPath &() const
{
return path;
}
/**
* Get the final component of the path (the input name).
* For a path like "a/b/c", returns "c".
*/
const FlakeId & inputName() const
{
return path.back();
}
/**
* Get the parent path (all components except the last).
* For a path like "a/b/c", returns "a/b".
*/
InputAttrPath parent() const
{
InputAttrPath result = path;
result.pop_back();
return result;
}
auto operator<=>(const NonEmptyInputAttrPath & other) const = default;
};
struct LockedNode;
/**
-13
View File
@@ -316,19 +316,6 @@ InputAttrPath parseInputAttrPath(std::string_view s)
return path;
}
std::optional<NonEmptyInputAttrPath> NonEmptyInputAttrPath::parse(std::string_view s)
{
auto path = parseInputAttrPath(s);
return make(std::move(path));
}
std::optional<NonEmptyInputAttrPath> NonEmptyInputAttrPath::make(InputAttrPath path)
{
if (path.empty())
return std::nullopt;
return NonEmptyInputAttrPath{std::move(path)};
}
std::map<InputAttrPath, Node::Edge> LockFile::getAllInputs() const
{
std::set<ref<Node>> done;
@@ -581,7 +581,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
goal.worker.childTerminated(&goal);
}
std::filesystem::path openLogFile() override
Path openLogFile() override
{
return goal.openLogFile();
}
@@ -602,9 +602,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
StorePathSet closure;
for (auto & i : defaultPathsInChroot)
try {
if (worker.store.isInStore(i.second.source.string()))
worker.store.computeFSClosure(
worker.store.toStorePath(i.second.source.string()).first, closure);
if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
} catch (InvalidPath & e) {
} catch (Error & e) {
e.addTrace({}, "while processing sandbox path '%s'", i.second.source);
+1 -3
View File
@@ -154,9 +154,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
}
if (buildMode == bmCheck && !allValid)
throw Error(
"some outputs of '%s' are not valid, so checking is not possible\n"
"Hint: --rebuild and --check error if the derivation was not previously built and cannot be substituted.\n"
" Remove it to perform a fresh build, or use --repair to rewrite missing or corrupted builds in the store.",
"some outputs of '%s' are not valid, so checking is not possible",
worker.store.printStorePath(drvPath));
}
+1 -1
View File
@@ -219,7 +219,7 @@ Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & g : goal->waitees) {
g->waiters.erase(goal);
g->waiters.extract(goal);
}
goal->waitees.clear();
-11
View File
@@ -9,7 +9,6 @@
#include "nix/store/gc-store.hh"
#include "nix/store/log-store.hh"
#include "nix/store/indirect-root-store.hh"
#include "nix/store/remote-store.hh"
#include "nix/store/path-with-outputs.hh"
#include "nix/util/finally.hh"
#include "nix/util/archive.hh"
@@ -1027,16 +1026,6 @@ void processConnection(ref<Store> store, FdSource && from, FdSink && to, Trusted
auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
(void) monitor; // suppress warning
ReceiveInterrupts receiveInterrupts;
/* When interrupted (e.g., SSH client disconnects), shutdown any downstream
store connections to break circular waits. This fixes deadlocks where the
daemon is waiting for a response from a downstream store while the downstream
is waiting for more data from this daemon. */
auto shutdownStoreOnInterrupt = createInterruptCallback([&store]() {
if (auto remoteStore = dynamic_cast<RemoteStore *>(&*store)) {
remoteStore->shutdownConnections();
}
});
#endif
/* Exchange the greeting. */
+1
View File
@@ -523,6 +523,7 @@ Derivation parseDerivation(
*/
static void printString(std::string & res, std::string_view s)
{
res.reserve(res.size() + s.size() * 2 + 2);
res += '"';
static constexpr auto chunkSize = 1024;
std::array<char, 2 * chunkSize + 2> buffer;
+1 -3
View File
@@ -5,7 +5,6 @@
#include "nix/util/finally.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/util/signals.hh"
#include "nix/util/serialise.hh"
#include "nix/util/util.hh"
#include "nix/store/posix-fs-canonicalise.hh"
@@ -577,10 +576,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1)
panic("Could not set non-blocking flag on client socket");
FdSource source(fdClient.get());
while (true) {
try {
auto path = source.readLine();
auto path = readLine(fdClient.get());
auto storePath = maybeParseStorePath(path);
if (storePath) {
debug("got new GC root '%s'", path);
+2 -13
View File
@@ -335,16 +335,7 @@ void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & cat
});
}
void to_json(nlohmann::json & j, const ChrootPath & cp)
{
j = nlohmann::json{{"source", cp.source.string()}, {"optional", cp.optional}};
}
void from_json(const nlohmann::json & j, ChrootPath & cp)
{
cp.source = j.at("source").get<std::string>();
cp.optional = j.at("optional").get<bool>();
}
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ChrootPath, source, optional)
template<>
PathsInChroot BaseSetting<PathsInChroot>::parse(const std::string & str) const
@@ -377,9 +368,7 @@ std::string BaseSetting<PathsInChroot>::to_string() const
{
std::vector<std::string> accum;
for (auto & [name, cp] : value) {
auto nameStr = name.string();
auto sourceStr = cp.source.string();
std::string s = name == cp.source ? nameStr : nameStr + "=" + sourceStr;
std::string s = name == cp.source ? name : name + "=" + cp.source;
if (cp.optional)
s += "?";
accum.push_back(std::move(s));
+1 -1
View File
@@ -182,7 +182,7 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
/* path is not a path, but a full relative or absolute
URL, e.g. we've seen in the wild NARINFO files have a URL
field which is
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=wvx0nans273vb7b0cjlplsmr2z905hwd`
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj`
(note the query param) and that gets passed here. */
auto result = parseURLRelative(path, cacheUriWithTrailingSlash);
@@ -38,9 +38,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
const Setting<std::string> secretKeyFiles{
this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."};
const Setting<std::optional<std::filesystem::path>> localNarCache{
const Setting<Path> localNarCache{
this,
std::nullopt,
"",
"local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
@@ -1,7 +1,6 @@
#pragma once
///@file
#include <filesystem>
#include <nlohmann/json_fwd.hpp>
#include "nix/store/build-result.hh"
@@ -44,11 +43,11 @@ struct BuilderFailureError : BuildError
*/
struct ChrootPath
{
std::filesystem::path source;
Path source;
bool optional = false;
};
typedef std::map<std::filesystem::path, ChrootPath> PathsInChroot; // maps target path to source path
typedef std::map<Path, ChrootPath> PathsInChroot; // maps target path to source path
/**
* Parameters by (mostly) `const` reference for `DerivationBuilder`.
@@ -111,7 +110,7 @@ struct DerivationBuilderCallbacks
/**
* Open a log file and a pipe to it.
*/
virtual std::filesystem::path openLogFile() = 0;
virtual Path openLogFile() = 0;
/**
* Close the log file.
@@ -186,7 +185,7 @@ struct DerivationBuilder : RestrictionContext
struct ExternalBuilder
{
StringSet systems;
std::filesystem::path program;
Path program;
std::vector<std::string> args;
};
+5 -5
View File
@@ -1125,11 +1125,11 @@ public:
character.
Example:
`/nix/store/l88brggg9hpy96ijds34dlq4n8fan63g-bash-4.4-p23-dev
/nix/store/vch71bhyi5akr5zs40k8h2wqxx69j80l-bash-4.4-p23-doc
/nix/store/c5cxjywi66iwn9dcx5yvwjkvl559ay6p-bash-4.4-p23-info
/nix/store/scz72lskj03ihkcn42ias5mlp4i4gr1k-bash-4.4-p23-man
/nix/store/a724znygmd1cac856j3gfsyvih3lw07j-bash-4.4-p23`.
`/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
/nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
/nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
/nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
/nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
)"};
Setting<unsigned int> downloadSpeed{
@@ -15,13 +15,13 @@ class RemoteFSAccessor : public SourceAccessor
bool requireValidPath;
std::optional<std::filesystem::path> cacheDir;
Path cacheDir;
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
friend struct BinaryCacheStore;
std::filesystem::path makeCacheFile(std::string_view hashPart, const std::string & ext);
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
@@ -33,7 +33,7 @@ public:
std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
RemoteFSAccessor(
ref<Store> store, bool requireValidPath = true, std::optional<std::filesystem::path> cacheDir = {});
ref<Store> store, bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = "");
std::optional<Stat> maybeLstat(const CanonPath & path) override;
@@ -2,12 +2,9 @@
///@file
#include <limits>
#include <set>
#include <string>
#include "nix/store/store-api.hh"
#include "nix/util/sync.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/store/gc-store.hh"
#include "nix/store/log-store.hh"
@@ -156,13 +153,6 @@ struct RemoteStore : public virtual Store, public virtual GcStore, public virtua
void flushBadConnections();
/**
* Shutdown all connections (both idle and in-use) to break any blocking I/O.
* This is called on interrupt to allow graceful termination when the client
* disconnects during a long-running operation.
*/
void shutdownConnections();
struct Connection;
ref<Connection> openConnectionWrapper();
@@ -201,12 +191,6 @@ private:
std::atomic_bool failed{false};
/**
* Track all active connection file descriptors (both idle and in-use).
* Used by shutdownConnections() to break blocking I/O on interrupt.
*/
Sync<std::set<Descriptor>> connectionFds;
void copyDrvsFromEvalStore(const std::vector<DerivedPath> & paths, std::shared_ptr<Store> evalStore);
};
+2 -4
View File
@@ -56,11 +56,9 @@ protected:
void upsertFile(
const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override
{
auto path2 = std::filesystem::path{config->binaryCacheDir} / path;
auto path2 = config->binaryCacheDir + "/" + path;
static std::atomic<int> counter{0};
createDirs(path2.parent_path());
auto tmp = path2;
tmp += fmt(".tmp.%d.%d", getpid(), ++counter);
Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);
AutoDelete del(tmp, false);
writeFile(tmp, source);
std::filesystem::rename(tmp, path2);
+9 -50
View File
@@ -1,3 +1,5 @@
#include "nix/util/archive.hh"
#include "nix/util/posix-source-accessor.hh"
#include "nix/store/store-api.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/globals.hh"
@@ -38,14 +40,13 @@ LocalFSStore::LocalFSStore(const Config & config)
{
}
struct LocalStoreAccessor : SourceAccessor
struct LocalStoreAccessor : PosixSourceAccessor
{
ref<SourceAccessor> accessor;
ref<LocalFSStore> store;
bool requireValidPath;
LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
: accessor(makeFSSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}))
: PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})
, store(store)
, requireValidPath(requireValidPath)
{
@@ -66,67 +67,25 @@ struct LocalStoreAccessor : SourceAccessor
return Stat{.type = tDirectory};
requireStoreObject(path);
return accessor->maybeLstat(path);
}
Stat lstat(const CanonPath & path) override
{
/* Also allow `path` to point to the entire store, which is
needed for resolving symlinks. */
if (path.isRoot())
return Stat{.type = tDirectory};
requireStoreObject(path);
return accessor->lstat(path);
return PosixSourceAccessor::maybeLstat(path);
}
DirEntries readDirectory(const CanonPath & path) override
{
requireStoreObject(path);
return accessor->readDirectory(path);
}
std::string readFile(const CanonPath & path) override
{
requireStoreObject(path);
return accessor->readFile(path);
return PosixSourceAccessor::readDirectory(path);
}
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
{
requireStoreObject(path);
return accessor->readFile(path, sink, sizeCallback);
return PosixSourceAccessor::readFile(path, sink, sizeCallback);
}
std::string readLink(const CanonPath & path) override
{
requireStoreObject(path);
return accessor->readLink(path);
}
std::string showPath(const CanonPath & path) override
{
return accessor->showPath(path);
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
{
return accessor->getPhysicalPath(path);
}
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
{
return accessor->getFingerprint(path);
}
std::optional<time_t> getLastModified() override
{
return accessor->getLastModified();
}
bool pathExists(const CanonPath & path) override
{
return accessor->pathExists(path);
return PosixSourceAccessor::readLink(path);
}
};
@@ -150,7 +109,7 @@ std::shared_ptr<SourceAccessor> LocalFSStore::getFSAccessor(const StorePath & pa
if (!pathExists(absPath))
return nullptr;
}
return makeFSSourceAccessor(std::move(absPath));
return std::make_shared<PosixSourceAccessor>(std::move(absPath));
}
const std::string LocalFSStore::drvsLogDir = "drvs";
+4 -3
View File
@@ -1059,7 +1059,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
if (info.ca) {
auto & specified = *info.ca;
auto actualHash = ({
SourcePath sourcePath = requireStoreObjectAccessor(info.path, /*requireValidPath=*/false);
auto accessor = getFSAccessor(false);
CanonPath path{info.path.to_string()};
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
auto fim = specified.method.getFileIngestionMethod();
switch (fim) {
@@ -1069,12 +1070,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
specified.hash.algo,
std::string{info.path.hashPart()},
};
dumpPath(sourcePath, caSink, (FileSerialisationMethod) fim);
dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim);
h = caSink.finish().hash;
break;
}
case FileIngestionMethod::Git:
h = git::dumpHash(specified.hash.algo, sourcePath).hash;
h = git::dumpHash(specified.hash.algo, {accessor, path}).hash;
break;
}
ContentAddress{
+9 -23
View File
@@ -23,9 +23,9 @@ void Store::computeFSClosure(
bool includeOutputs,
bool includeDerivers)
{
std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
std::function<asio::awaitable<StorePathSet>(const StorePath & path)> queryDeps;
if (flipDirection)
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
StorePathSet res;
StorePathSet referrers;
queryReferrers(path, referrers);
@@ -41,12 +41,14 @@ void Store::computeFSClosure(
for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
if (maybeOutPath && isValidPath(*maybeOutPath))
res.insert(*maybeOutPath);
return res;
co_return res;
};
else
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
StorePathSet res;
auto info = fut.get();
auto info = co_await callbackToAwaitable<ref<const ValidPathInfo>>(
[this, path](Callback<ref<const ValidPathInfo>> cb) { queryPathInfo(path, std::move(cb)); });
for (auto & ref : info->references)
if (ref != path)
res.insert(ref);
@@ -58,25 +60,9 @@ void Store::computeFSClosure(
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
res.insert(*info->deriver);
return res;
co_return res;
};
computeClosure<StorePath>(
startPaths,
paths_,
[&](const StorePath & path, std::function<void(std::promise<std::set<StorePath>> &)> processEdges) {
std::promise<std::set<StorePath>> promise;
std::function<void(std::future<ref<const ValidPathInfo>>)> getDependencies =
[&](std::future<ref<const ValidPathInfo>> fut) {
try {
promise.set_value(queryDeps(path, fut));
} catch (...) {
promise.set_exception(std::current_exception());
}
};
queryPathInfo(path, getDependencies);
processEdges(promise);
});
computeClosure<StorePath>(startPaths, paths_, queryDeps);
}
void Store::computeFSClosure(
+11 -14
View File
@@ -8,27 +8,24 @@
namespace nix {
RemoteFSAccessor::RemoteFSAccessor(
ref<Store> store, bool requireValidPath, std::optional<std::filesystem::path> cacheDir_)
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, const Path & cacheDir)
: store(store)
, requireValidPath(requireValidPath)
, cacheDir(std::move(cacheDir_))
, cacheDir(cacheDir)
{
if (cacheDir)
createDirs(*cacheDir);
if (cacheDir != "")
createDirs(cacheDir);
}
std::filesystem::path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{
assert(cacheDir);
auto res = (*cacheDir / hashPart);
res.concat(concatStrings(".", ext));
return res;
assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
{
if (cacheDir) {
if (cacheDir != "") {
try {
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(hashPart, "nar"), nar);
@@ -40,7 +37,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
auto narAccessor = makeNarAccessor(std::move(nar));
nars.emplace(hashPart, narAccessor);
if (cacheDir) {
if (cacheDir != "") {
try {
nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
writeFile(makeCacheFile(hashPart, "ls"), j.dump());
@@ -67,9 +64,9 @@ std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath &
return i->second;
std::string listing;
std::filesystem::path cacheFile;
Path cacheFile;
if (cacheDir && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try {
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
-20
View File
@@ -19,10 +19,6 @@
#include "nix/store/filetransfer.hh"
#include "nix/util/signals.hh"
#ifndef _WIN32
# include <sys/socket.h>
#endif
#include <nlohmann/json.hpp>
namespace nix {
@@ -42,8 +38,6 @@ RemoteStore::RemoteStore(const Config & config)
failed = true;
throw;
}
/* Track the connection FD for shutdownConnections() */
connectionFds.lock()->insert(conn->from.fd);
return conn;
},
[this](const ref<Connection> & r) {
@@ -803,20 +797,6 @@ void RemoteStore::flushBadConnections()
connections->flushBad();
}
void RemoteStore::shutdownConnections()
{
#ifndef _WIN32
auto fds = connectionFds.lock();
for (auto fd : *fds) {
/* Use shutdown() instead of close() to signal EOF to any blocking
reads/writes without actually closing the FD (which would cause
issues if the connection is still in use). This breaks circular
waits when the client disconnects during long-running operations. */
::shutdown(fromDescriptorReadOnly(fd), SHUT_RDWR);
}
#endif
}
void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
{
auto conn(getConnection());
Loaded 100 of 221 files, more files were not shown because too many files have changed in this diff. Show more