Compare commits

..
Author SHA1 Message Date
Domen Kožar b8cb447a45 use newer macos to see if it helps 2023-06-28 13:42:35 +01:00
Domen Kožar 789d1cf409 flake: use nixos-unstable 2023-06-28 12:48:54 +01:00
186 changed files with 1315 additions and 3530 deletions

No files matched your search

+1 -1
View File
@@ -16,7 +16,7 @@
"new-cli":
- src/nix/**/*
"with-tests":
"tests":
# Unit tests
- src/*/tests/**/*
# Functional and integration tests
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
os: [ubuntu-latest, macos-13]
runs-on: ${{ matrix.os }}
timeout-minutes: 60
steps:
+1 -3
View File
@@ -18,7 +18,7 @@ perl/Makefile.config
/doc/manual/generated/*
/doc/manual/nix.json
/doc/manual/conf-file.json
/doc/manual/language.json
/doc/manual/builtins.json
/doc/manual/xp-features.json
/doc/manual/src/SUMMARY.md
/doc/manual/src/command-ref/new-cli
@@ -26,7 +26,6 @@ perl/Makefile.config
/doc/manual/src/command-ref/experimental-features-shortlist.md
/doc/manual/src/contributing/experimental-feature-descriptions.md
/doc/manual/src/language/builtins.md
/doc/manual/src/language/builtin-constants.md
# /scripts/
/scripts/nix-profile.sh
@@ -95,7 +94,6 @@ perl/Makefile.config
# /tests/lang/
/tests/lang/*.out
/tests/lang/*.out.xml
/tests/lang/*.err
/tests/lang/*.ast
/perl/lib/Nix/Config.pm
-2
View File
@@ -27,8 +27,6 @@ makefiles += \
src/libstore/tests/local.mk \
src/libexpr/tests/local.mk \
tests/local.mk \
tests/ca/local.mk \
tests/dyn-drv/local.mk \
tests/test-libstoreconsumer/local.mk \
tests/plugins/local.mk
else
+1 -8
View File
@@ -5,14 +5,7 @@ AC_CONFIG_AUX_DIR(config)
AC_PROG_SED
# Construct a Nix system name (like "i686-linux"):
# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1
# The inital value is produced by the `config/config.guess` script:
# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess
# It has the following form, which is not documented anywhere:
# <cpu>-<vendor>-<os>[<version>][-<abi>]
# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead:
# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub
# Construct a Nix system name (like "i686-linux").
AC_CANONICAL_HOST
AC_MSG_CHECKING([for the canonical Nix system name])
-31
View File
@@ -1,31 +0,0 @@
let
inherit (builtins) concatStringsSep attrValues mapAttrs;
inherit (import ./utils.nix) optionalString squash;
in
builtinsInfo:
let
showBuiltin = name: { doc, type, impure-only }:
let
type' = optionalString (type != null) " (${type})";
impureNotice = optionalString impure-only ''
> **Note**
>
> Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
'';
in
squash ''
<dt id="builtins-${name}">
<a href="#builtins-${name}"><code>${name}</code></a>${type'}
</dt>
<dd>
${doc}
${impureNotice}
</dd>
'';
in
concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo))
+7 -11
View File
@@ -1,28 +1,24 @@
let
inherit (builtins) concatStringsSep attrValues mapAttrs;
inherit (import ./utils.nix) optionalString squash;
inherit (builtins) concatStringsSep attrNames;
in
builtinsInfo:
let
showBuiltin = name: { doc, args, arity, experimental-feature }:
showBuiltin = name:
let
experimentalNotice = optionalString (experimental-feature != null) ''
This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled.
'';
inherit (builtinsInfo.${name}) doc args;
in
squash ''
''
<dt id="builtins-${name}">
<a href="#builtins-${name}"><code>${name} ${listArgs args}</code></a>
</dt>
<dd>
${doc}
${experimentalNotice}
${doc}
</dd>
'';
listArgs = args: concatStringsSep " " (map (s: "<var>${s}</var>") args);
in
concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo))
concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo))
+5 -11
View File
@@ -128,20 +128,14 @@ $(d)/xp-features.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
@mv $@.tmp $@
$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@cat doc/manual/src/language/builtins-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp;
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp;
@cat doc/manual/src/language/builtins-suffix.md >> $@.tmp
@mv $@.tmp $@
$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix
@cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp
$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp;
@cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp
@mv $@.tmp $@
$(d)/language.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp
$(d)/builtins.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp
@mv $@.tmp $@
# Generate the HTML manual.
@@ -173,7 +167,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
done
@touch $@
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md
$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md
$(trace-gen) \
tmp="$$(mktemp -d)"; \
cp -r doc/manual "$$tmp"; \
+6 -14
View File
@@ -281,7 +281,7 @@ const redirects = {
"chap-introduction": "introduction.html",
"ch-basic-package-mgmt": "package-management/basic-package-mgmt.html",
"ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html",
"sec-channels": "command-ref/nix-channel.html",
"sec-channels": "package-management/channels.html",
"ssec-copy-closure": "package-management/copy-closure.html",
"sec-garbage-collection": "package-management/garbage-collection.html",
"ssec-gc-roots": "package-management/garbage-collector-roots.html",
@@ -330,31 +330,23 @@ const redirects = {
"ssec-relnotes-2.0": "release-notes/rl-2.0.html",
"ssec-relnotes-2.1": "release-notes/rl-2.1.html",
"ssec-relnotes-2.2": "release-notes/rl-2.2.html",
"ssec-relnotes-2.3": "release-notes/rl-2.3.html",
"ssec-relnotes-2.3": "release-notes/rl-2.3.html"
},
"language/values.html": {
"simple-values": "#primitives",
"lists": "#list",
"strings": "#string",
"lists": "#list",
"attribute-sets": "#attribute-set",
"attribute-sets": "#attribute-set"
},
"installation/installing-binary.html": {
"linux": "uninstall.html#linux",
"macos": "uninstall.html#macos",
"uninstalling": "uninstall.html",
"uninstalling": "uninstall.html"
}
"contributing/hacking.html": {
"nix-with-flakes": "#building-nix-with-flakes",
"classic-nix": "#building-nix",
"running-tests": "testing.html#running-tests",
"unit-tests": "testing.html#unit-tests",
"functional-tests": "testing.html#functional-tests",
"debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests",
"integration-tests": "testing.html#integration-tests",
"installer-tests": "testing.html#installer-tests",
"one-time-setup": "testing.html#one-time-setup",
"using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing",
"nix-with-flakes": "#building-nix-with-flakes"
"classic-nix": "#building-nix"
}
};
+2 -2
View File
@@ -21,6 +21,7 @@
- [Profiles](package-management/profiles.md)
- [Garbage Collection](package-management/garbage-collection.md)
- [Garbage Collector Roots](package-management/garbage-collector-roots.md)
- [Channels](package-management/channels.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
@@ -103,12 +104,11 @@
- [Glossary](glossary.md)
- [Contributing](contributing/contributing.md)
- [Hacking](contributing/hacking.md)
- [Testing](contributing/testing.md)
- [Experimental Features](contributing/experimental-features.md)
- [CLI guideline](contributing/cli-guideline.md)
- [C++ style guide](contributing/cxx.md)
- [Release Notes](release-notes/release-notes.md)
- [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
- [Release X.Y (202?-??-??)](release-notes/rl-next.md)
- [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)
- [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
- [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
+30 -45
View File
@@ -8,46 +8,36 @@
# Description
Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version.
A Nix channel is a mechanism that allows you to automatically stay
up-to-date with a set of pre-built Nix expressions. A Nix channel is
just a URL that points to a place containing a set of Nix expressions.
The moving parts of channels are:
- The official channels listed at <https://nixos.org/channels>
- The user-specific list of [subscribed channels](#subscribed-channels)
- The [downloaded channel contents](#channels)
- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH)
> **Note**
>
> The state of a subscribed channel is external to the Nix expressions relying on it.
> This may limit reproducibility.
>
> Dependencies on other Nix expressions can be declared explicitly with:
> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions
> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations
To see the list of official NixOS channels, visit
<https://nixos.org/channels>.
This command has the following operations:
- `--add` *url* \[*name*\]\
Add a channel *name* located at *url* to the list of subscribed channels.
If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed.
> **Note**
>
> `--add` does not automatically perform an update.
> Use `--update` explicitly.
Adds a channel named *name* with URL *url* to the list of subscribed
channels. If *name* is omitted, it defaults to the last component of
*url*, with the suffixes `-stable` or `-unstable` removed.
A channel URL must point to a directory containing a file `nixexprs.tar.gz`.
At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point.
- `--remove` *name*\
Remove the channel *name* from the list of subscribed channels.
Removes the channel named *name* from the list of subscribed
channels.
- `--list`\
Print the names and URLs of all subscribed channels on standard output.
Prints the names and URLs of all subscribed channels on standard
output.
- `--update` \[*names*…\]\
Download the Nix expressions of subscribed channels and create a new generation.
Update all channels if none is specified, and only those included in *names* otherwise.
Downloads the Nix expressions of all subscribed channels (or only
those included in *names* if specified) and makes them the default
for `nix-env` operations (by symlinking them from the directory
`~/.nix-defexpr`).
- `--list-generations`\
Prints a list of all the current existing generations for the
@@ -59,8 +49,13 @@ This command has the following operations:
```
- `--rollback` \[*generation*\]\
Revert channels to the state before the last call to `nix-channel --update`.
Optionally, you can specify a specific channel *generation* number to restore.
Reverts the previous call to `nix-channel
--update`. Optionally, you can specify a specific channel generation
number to restore.
Note that `--add` does not automatically perform an update.
The list of subscribed channels is stored in `~/.nix-channels`.
{{#include ./opt-common.md}}
@@ -74,33 +69,23 @@ This command has the following operations:
# Examples
Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package:
To subscribe to the Nixpkgs channel and install the GNU Hello package:
```console
$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
$ nix-channel --list
nixpkgs https://nixos.org/channels/nixpkgs
$ nix-channel --update
$ nix-shell -p hello --run hello
hello
$ nix-env --install --attr nixpkgs.hello
```
Revert channel updates using `--rollback`:
You can revert channel updates using `--rollback`:
```console
$ nix-instantiate --eval '<nixpkgs>' --attr lib.version
"22.11pre296212.530a53dcbc9"
$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version'
"14.04.527.0e935f1"
$ nix-channel --rollback
switching from generation 483 to 482
$ nix-instantiate --eval '<nixpkgs>' --attr lib.version
"22.11pre281526.d0419badfad"
```
Remove a channel:
```console
$ nix-channel --remove nixpkgs
$ nix-channel --list
$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version'
"14.04.526.dbadfad"
```
+182 -48
View File
@@ -110,72 +110,41 @@ You can also build Nix for one of the [supported platforms](#platforms).
## Platforms
Nix can be built for various platforms, as specified in [`flake.nix`]:
As specified in [`flake.nix`], Nix can be built for various platforms:
- `aarch64-linux`
- `i686-linux`
- `x86_64-darwin`
- `x86_64-linux`
[`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix
- `x86_64-linux`
- `x86_64-darwin`
- `i686-linux`
- `aarch64-linux`
- `aarch64-darwin`
- `armv6l-linux`
- `armv7l-linux`
In order to build Nix for a different platform than the one you're currently
on, you need a way for your current Nix installation to build code for that
platform. Common solutions include [remote builders] and [binary format emulation]
on, you need to have some way for your system Nix to build code for that
platform. Common solutions include [remote builders] and [binfmt emulation]
(only supported on NixOS).
[remote builders]: ../advanced-topics/distributed-builds.md
[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems
[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems
Given such a setup, executing the build only requires selecting the respective attribute.
For example, to compile for `aarch64-linux`:
```console
$ nix-build --attr packages.aarch64-linux.default
```
or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled:
These solutions let Nix perform builds as if you're on the native platform, so
executing the build is as simple as
```console
$ nix build .#packages.aarch64-linux.default
```
Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
for flake-enabled Nix, or
## System type
Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
```
<cpu>-<os>[-<abi>]
```console
$ nix-build --attr packages.aarch64-linux.default
```
It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)):
for classic Nix.
```
<cpu>-<vendor>-<os>[<version>][-<abi>]
```
You can use any of the other supported platforms in place of `aarch64-linux`.
When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)):
```
<cpu>-<vendor>[-<kernel>]-<os>
```
For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows:
| `config.guess` | Nix |
|----------------------------|---------------------|
| `amd64` | `x86_64` |
| `i*86` | `i686` |
| `arm6` | `arm6l` |
| `arm7` | `arm7l` |
| `linux-gnu*` | `linux` |
| `linux-musl*` | `linux` |
Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`.
## Compilation environments
@@ -221,6 +190,171 @@ Configure your editor to use the `clangd` from the shell, either by running it i
> Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim).
> Editor-specific setup is typically opinionated, so we will not cover it here in more detail.
## Running tests
### Unit-tests
The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
under `src/{library_name}/tests` using the
[googletest](https://google.github.io/googletest/) and
[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
### Functional tests
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
Each test is a bash script.
The whole test suite can be run with:
```shell-session
$ make install && make installcheck
ran test tests/foo.sh... [PASS]
ran test tests/bar.sh... [PASS]
...
```
Individual tests can be run with `make`:
```shell-session
$ make tests/${testName}.sh.test
ran test tests/${testName}.sh... [PASS]
```
or without `make`:
```shell-session
$ ./mk/run-test.sh tests/${testName}.sh
ran test tests/${testName}.sh... [PASS]
```
To see the complete output, one can also run:
```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh
+ foo
output from foo
+ bar
output from bar
...
```
The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
#### Debugging failing functional tests
When a functional test fails, it usually does so somewhere in the middle of the script.
To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
For example, if the script looks like:
```bash
foo
nix blah blub
bar
```
edit it like so:
```diff
foo
-nix blah blub
+gdb --args nix blah blub
bar
```
Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh
...
+ gdb blash blub
GNU gdb (GDB) 12.1
...
(gdb)
```
One can debug the Nix invocation in all the usual ways.
For example, enter `run` to start the Nix invocation.
### Integration tests
The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
### Installer tests
After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.
Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91):
- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache:
- `x86_64-linux`
- `armv6l-linux`
- `armv7l-linux`
- `x86_64-darwin`
- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
#### One-time setup
1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix).
2. At cachix.org:
- Create or log in to an account.
- Create a Cachix cache using the format `<github-username>-nix-install-tests`.
- Navigate to the new cache > Settings > Auth Tokens.
- Generate a new Cachix auth token and copy the generated value.
3. At github.com:
- Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret.
- Name the secret `CACHIX_AUTH_TOKEN`.
- Paste the copied value of the Cachix cache auth token.
#### Using the CI-generated installer for manual testing
After the CI run completes, you can check the output to extract the installer URL:
1. Click into the detailed view of the CI run.
2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them).
3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`)
4. Copy the value of `install_url`
5. To generate an install command, plug this `install_url` and your GitHub username into this template:
```console
curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
```
<!-- #### Manually generating test installers
There's obviously a manual way to do this, and it's still the only way for
platforms that lack GA runners.
I did do this back in Fall 2020 (before the GA approach encouraged here). I'll
sketch what I recall in case it encourages someone to fill in detail, but: I
didn't know what I was doing at the time and had to fumble/ask around a lot--
so I don't want to uphold any of it as "right". It may have been dumb or
the _hard_ way from the getgo. Fundamentals may have changed since.
Here's the build command I used to do this on and for x86_64-darwin:
nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball"
I used the stable out-link to make it easier to script the next steps:
link=$(readlink /tmp/foo)
cp $link/*-darwin.tar.xz ~/somewheres
I've lost the last steps and am just going from memory:
From here, I think I had to extract and modify the `install` script to point
it at this tarball (which I scped to my own site, but it might make more sense
to just share them locally). I extracted this script once and then just
search/replaced in it for each new build.
The installer now supports a `--tarball-url-prefix` flag which _may_ have
solved this need?
-->
### Checking links in the manual
The build checks for broken internal links.
-221
View File
@@ -1,221 +0,0 @@
# Running tests
## Unit-tests
The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
under `src/{library_name}/tests` using the
[googletest](https://google.github.io/googletest/) and
[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks.
You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
## Functional tests
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
Each test is a bash script.
### Running the whole test suite
The whole test suite can be run with:
```shell-session
$ make install && make installcheck
ran test tests/foo.sh... [PASS]
ran test tests/bar.sh... [PASS]
...
```
### Grouping tests
Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
Each test group is in a subdirectory of `tests`.
For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
That test group can be run like this:
```shell-session
$ make ca.test-group -j50
ran test tests/ca/nix-run.sh... [PASS]
ran test tests/ca/import-derivation.sh... [PASS]
...
```
The test group is defined in Make like this:
```makefile
$(test-group-name)-tests := \
$(d)/test0.sh \
$(d)/test1.sh \
...
install-tests-groups += $(test-group-name)
```
### Running individual tests
Individual tests can be run with `make`:
```shell-session
$ make tests/${testName}.sh.test
ran test tests/${testName}.sh... [PASS]
```
or without `make`:
```shell-session
$ ./mk/run-test.sh tests/${testName}.sh
ran test tests/${testName}.sh... [PASS]
```
To see the complete output, one can also run:
```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh
+ foo
output from foo
+ bar
output from bar
...
```
The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails.
### Debugging failing functional tests
When a functional test fails, it usually does so somewhere in the middle of the script.
To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB.
For example, if the script looks like:
```bash
foo
nix blah blub
bar
```
edit it like so:
```diff
foo
-nix blah blub
+gdb --args nix blah blub
bar
```
Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
```shell-session
$ ./mk/debug-test.sh tests/${testName}.sh
...
+ gdb blash blub
GNU gdb (GDB) 12.1
...
(gdb)
```
One can debug the Nix invocation in all the usual ways.
For example, enter `run` to start the Nix invocation.
### Characterization testing
Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests.
This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward.
For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered.
It is frequently useful to regenerate the expected output.
To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`.
(At least, this is the convention we've used for `tests/lang.sh`.
If we add more characterization testing we should always strive to be consistent.)
An interesting situation to document is the case when these tests are "overfitted".
The language tests are, again, an example of this.
The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language.
However, the errors and warnings during evaluation (successful or not) are not stable in this way.
We are free to change how they are displayed at any time.
It may be surprising that we would test non-normative behavior like diagnostic outputs.
Diagnostic outputs are indeed not a stable interface, but they still are important to users.
By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review.
Regressions are caught, and improvements always show up in code review.
To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`.
## Integration tests
The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
## Installer tests
After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch.
Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91):
- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache:
- `x86_64-linux`
- `armv6l-linux`
- `armv7l-linux`
- `x86_64-darwin`
- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
### One-time setup
1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix).
2. At cachix.org:
- Create or log in to an account.
- Create a Cachix cache using the format `<github-username>-nix-install-tests`.
- Navigate to the new cache > Settings > Auth Tokens.
- Generate a new Cachix auth token and copy the generated value.
3. At github.com:
- Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret.
- Name the secret `CACHIX_AUTH_TOKEN`.
- Paste the copied value of the Cachix cache auth token.
## Working on documentation
### Using the CI-generated installer for manual testing
After the CI run completes, you can check the output to extract the installer URL:
1. Click into the detailed view of the CI run.
2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them).
3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`)
4. Copy the value of `install_url`
5. To generate an install command, plug this `install_url` and your GitHub username into this template:
```console
curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve
```
<!-- #### Manually generating test installers
There's obviously a manual way to do this, and it's still the only way for
platforms that lack GA runners.
I did do this back in Fall 2020 (before the GA approach encouraged here). I'll
sketch what I recall in case it encourages someone to fill in detail, but: I
didn't know what I was doing at the time and had to fumble/ask around a lot--
so I don't want to uphold any of it as "right". It may have been dumb or
the _hard_ way from the getgo. Fundamentals may have changed since.
Here's the build command I used to do this on and for x86_64-darwin:
nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball"
I used the stable out-link to make it easier to script the next steps:
link=$(readlink /tmp/foo)
cp $link/*-darwin.tar.xz ~/somewheres
I've lost the last steps and am just going from memory:
From here, I think I had to extract and modify the `install` script to point
it at this tarball (which I scped to my own site, but it might make more sense
to just share them locally). I extracted this script once and then just
search/replaced in it for each new build.
The installer now supports a `--tarball-url-prefix` flag which _may_ have
solved this need?
-->
@@ -1,5 +0,0 @@
# Built-in Constants
These constants are built into the Nix language evaluator:
<dl>
@@ -1 +0,0 @@
</dl>
@@ -0,0 +1,43 @@
# Built-in Constants
These constants are built into the Nix language evaluator:
- [`builtins`]{#builtins-builtins} (attribute set)
Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope.
Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
```nix
if builtins ? getEnv then builtins.getEnv "PATH" else ""
```
- [`builtins.currentSystem`]{#builtins-currentSystem} (string)
The built-in value `currentSystem` evaluates to the Nix platform
identifier for the Nix installation on which the expression is being
evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
- [`builtins.currentTime`]{#builtins-currentTime} (integer)
Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation.
Repeated references to that name will re-use the initially obtained value.
Example:
```console
$ nix repl
Welcome to Nix 2.15.1 Type :? for help.
nix-repl> builtins.currentTime
1683705525
nix-repl> builtins.currentTime
1683705525
```
The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation.
Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
+1 -1
View File
@@ -3,7 +3,7 @@
This section lists the functions built into the Nix language evaluator.
All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant.
For convenience, some built-ins can be accessed directly:
For convenience, some built-ins are can be accessed directly:
- [`derivation`](#builtins-derivation)
- [`import`](#builtins-import)
+20 -31
View File
@@ -92,10 +92,10 @@ In this fragment from `all-packages.nix`,
```nix
graphviz = (import ../tools/graphics/graphviz) {
inherit fetchurl stdenv libpng libjpeg expat x11 yacc;
inherit (xorg) libXaw;
inherit (xlibs) libXaw;
};
xorg = {
xlibs = {
libX11 = ...;
libXaw = ...;
...
@@ -109,7 +109,7 @@ libjpg = ...;
the set used in the function call to the function defined in
`../tools/graphics/graphviz` inherits a number of variables from the
surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw`
(the X Athena Widgets) from the `xorg` set.
(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set.
Summarizing the fragment
@@ -208,41 +208,30 @@ three kinds of patterns:
```nix
{ x, y, z, ... } @ args: z + y + x + args.a
```
Here `args` is bound to the argument *as passed*, which is further
matched against the pattern `{ x, y, z, ... }`.
The `@`-pattern makes mainly sense with an ellipsis(`...`) as
Here `args` is bound to the entire argument, which is further
matched against the pattern `{ x, y, z,
... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as
you can access attribute names as `a`, using `args.a`, which was
given as an additional attribute to the function.
> **Warning**
>
> `args@` binds the name `args` to the attribute set that is passed to the function.
> In particular, `args` does *not* include any default values specified with `?` in the function's set pattern.
>
>
> The `args@` expression is bound to the argument passed to the
> function which means that attributes with defaults that aren't
> explicitly specified in the function call won't cause an
> evaluation error, but won't exist in `args`.
>
> For instance
>
>
> ```nix
> let
> f = args@{ a ? 23, ... }: [ a args ];
> function = args@{ a ? 23, ... }: args;
> in
> f {}
> ```
>
> is equivalent to
>
> ```nix
> let
> f = args @ { ... }: [ (args.a or 23) args ];
> in
> f {}
> ```
>
> and both expressions will evaluate to:
>
> ```nix
> [ 23 {} ]
> ```
> function {}
> ````
>
> will evaluate to an empty attribute set.
Note that functions do not have names. If you want to give them a name,
you can bind them to an attribute, e.g.,
@@ -25,7 +25,7 @@ or completely new ones.)
You can manually download the latest version of Nixpkgs from
<https://github.com/NixOS/nixpkgs>. However, it’s much more
convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes
convenient to use the Nixpkgs [*channel*](channels.md), since it makes
it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
automatically added to your list of “subscribed” channels when you
install Nix. If this is not the case for some reason, you can add it
@@ -0,0 +1,50 @@
# Channels
If you want to stay up to date with a set of packages, it’s not very
convenient to manually download the latest set of Nix expressions for
those packages and upgrade using `nix-env`. Fortunately, there’s a
better way: *Nix channels*.
A Nix channel is just a URL that points to a place that contains a set
of Nix expressions and a manifest. Using the command
[`nix-channel`](../command-ref/nix-channel.md) you can automatically
stay up to date with whatever is available at that URL.
To see the list of official NixOS channels, visit
<https://nixos.org/channels>.
You can “subscribe” to a channel using `nix-channel --add`, e.g.,
```console
$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
```
subscribes you to a channel that always contains that latest version of
the Nix Packages collection. (Subscribing really just means that the URL
is added to the file `~/.nix-channels`, where it is read by subsequent
calls to `nix-channel
--update`.) You can “unsubscribe” using `nix-channel
--remove`:
```console
$ nix-channel --remove nixpkgs
```
To obtain the latest Nix expressions available in a channel, do
```console
$ nix-channel --update
```
This downloads and unpacks the Nix expressions in every channel
(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of
each channel’s Nix expressions available by default to `nix-env`
operations (via the symlink `~/.nix-defexpr/channels`). Consequently,
you can then say
```console
$ nix-env --upgrade
```
to upgrade all packages in your profile to the latest versions available
in the subscribed channels.
-29
View File
@@ -1,29 +0,0 @@
# Release 2.17 (2023-07-24)
* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand.
* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure.
* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths.
Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths.
* Nested dynamic attributes are now merged correctly by the parser. For example:
```nix
{
nested = { foo = 1; };
nested = { ${"ba" + "r"} = 2; };
}
```
This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to:
```nix
{ nested = { bar = 2; foo = 1; }; }
```
Note that the feature of merging multiple attribute set declarations is of questionable value.
It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute.
This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon.
* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details.
+2
View File
@@ -1 +1,3 @@
# Release X.Y (202?-??-??)
- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand
Generated
+4 -4
View File
@@ -34,16 +34,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1670461440,
"narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=",
"lastModified": 1687898314,
"narHash": "sha256-B4BHon3uMXQw8ZdbwxRK1BmxVOGBV4viipKpGaIlGwk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425",
"rev": "e18dc963075ed115afb3e312b64643bf8fd4b474",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11-small",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
+2 -2
View File
@@ -1,7 +1,7 @@
{
description = "The purely functional package manager";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
@@ -11,7 +11,7 @@
let
inherit (nixpkgs) lib;
officialRelease = true;
officialRelease = false;
version = lib.fileContents ./.version + versionSuffix;
versionSuffix =
-1
View File
@@ -10,7 +10,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
KillMode=process
LimitNOFILE=1048576
TasksMax=1048576
[Install]
WantedBy=multi-user.target
+1 -19
View File
@@ -10,7 +10,6 @@ bin-scripts :=
noinst-scripts :=
man-pages :=
install-tests :=
install-tests-groups :=
ifdef HOST_OS
HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
@@ -122,16 +121,7 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
$(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
$(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
$(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
$(foreach test, $(install-tests), \
$(eval $(call run-install-test,$(test))) \
$(eval installcheck: $(test).test))
$(foreach test-group, $(install-tests-groups), \
$(eval $(call run-install-test-group,$(test-group))) \
$(eval installcheck: $(test-group).test-group) \
$(foreach test, $($(test-group)-tests), \
$(eval $(call run-install-test,$(test))) \
$(eval $(test-group).test-group: $(test).test)))
$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
$(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
@@ -161,14 +151,6 @@ ifdef libs-list
@echo "The following libraries can be built:"
@echo ""
@for i in $(libs-list); do echo " $$i"; done
endif
ifdef install-tests-groups
@echo ""
@echo "The following groups of functional tests can be run:"
@echo ""
@for i in $(install-tests-groups); do echo " $$i.test-group"; done
@echo ""
@echo "(installcheck includes tests in test groups too.)"
endif
@echo ""
@echo "The following variables control the build:"
+2 -6
View File
@@ -4,6 +4,8 @@ test-deps =
define run-install-test
installcheck: $1.test
.PHONY: $1.test
$1.test: $1 $(test-deps)
@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
@@ -14,12 +16,6 @@ define run-install-test
endef
define run-install-test-group
.PHONY: $1.test-group
endef
.PHONY: check installcheck
print-top-help += \
+4 -2
View File
@@ -294,8 +294,10 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
.method = method,
.hash = h,
.hash = {
.method = method,
.hash = h,
},
.references = {},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
-67
View File
@@ -1,67 +0,0 @@
#include "built-path.hh"
#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
#include <optional>
namespace nix {
nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = store->printStorePath(path);
}
return res;
}
StorePathSet BuiltPath::outPaths() const
{
return std::visit(
overloaded{
[](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
[](const BuiltPath::Built & b) {
StorePathSet res;
for (auto & [_, path] : b.outputs)
res.insert(path);
return res;
},
}, raw()
);
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
{
RealisedPath::Set res;
std::visit(
overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath));
for (auto& [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled(
Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath), outputName);
auto thisRealisation = store.queryRealisation(
DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // We’ve built it, so we must
// have the realisation
res.insert(*thisRealisation);
} else {
res.insert(outputPath);
}
}
},
},
raw());
return res;
}
}
-48
View File
@@ -1,48 +0,0 @@
#include "derived-path.hh"
#include "realisation.hh"
namespace nix {
/**
* A built derived path with hints in the form of optional concrete output paths.
*
* See 'BuiltPath' for more an explanation.
*/
struct BuiltPathBuilt {
StorePath drvPath;
std::map<std::string, StorePath> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static BuiltPathBuilt parse(const Store & store, std::string_view);
GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs);
};
using _BuiltPathRaw = std::variant<
DerivedPath::Opaque,
BuiltPathBuilt
>;
/**
* A built path. Similar to a DerivedPath, but enriched with the corresponding
* output path(s).
*/
struct BuiltPath : _BuiltPathRaw {
using Raw = _BuiltPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = BuiltPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
StorePathSet outPaths() const;
RealisedPath::Set toRealisedPaths(Store & store) const;
};
typedef std::vector<BuiltPath> BuiltPaths;
}
+1 -3
View File
@@ -105,9 +105,7 @@ MixEvalArgs::MixEvalArgs()
)",
.category = category,
.labels = {"path"},
.handler = {[&](std::string s) {
searchPath.elements.emplace_back(SearchPath::Elem::parse(s));
}}
.handler = {[&](std::string s) { searchPath.push_back(s); }}
});
addFlag({
+1 -2
View File
@@ -3,7 +3,6 @@
#include "args.hh"
#include "common-args.hh"
#include "search-path.hh"
namespace nix {
@@ -20,7 +19,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
Bindings * getAutoArgs(EvalState & state);
SearchPath searchPath;
Strings searchPath;
std::optional<std::string> evalStoreUrl;
+1 -1
View File
@@ -151,7 +151,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
},
ExtraPathInfoFlake::Flake {
.originalRef = flakeRef,
.lockedRef = getLockedFlake()->flake.lockedRef,
.resolvedRef = getLockedFlake()->flake.lockedRef,
}),
}};
}
+1 -1
View File
@@ -19,7 +19,7 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue
*/
struct Flake {
FlakeRef originalRef;
FlakeRef lockedRef;
FlakeRef resolvedRef;
};
Flake flake;
-1
View File
@@ -5,7 +5,6 @@
#include "path.hh"
#include "outputs-spec.hh"
#include "derived-path.hh"
#include "built-path.hh"
#include "store-api.hh"
#include "build-result.hh"
+4 -4
View File
@@ -68,7 +68,7 @@ struct NixRepl
const Path historyFile;
NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state,
NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state,
std::function<AnnotatedValues()> getValues);
virtual ~NixRepl();
@@ -104,7 +104,7 @@ std::string removeWhitespace(std::string s)
}
NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<NixRepl::AnnotatedValues()> getValues)
: AbstractNixRepl(state)
, debugTraceIndex(0)
@@ -1024,7 +1024,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<AnnotatedValues()> getValues)
{
return std::make_unique<NixRepl>(
@@ -1044,7 +1044,7 @@ void AbstractNixRepl::runSimple(
NixRepl::AnnotatedValues values;
return values;
};
SearchPath searchPath = {};
const Strings & searchPath = {};
auto repl = std::make_unique<NixRepl>(
searchPath,
openStore(),
+1 -1
View File
@@ -25,7 +25,7 @@ struct AbstractNixRepl
typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
static std::unique_ptr<AbstractNixRepl> create(
const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<AnnotatedValues()> getValues);
static void runSimple(
+44 -60
View File
@@ -95,16 +95,11 @@ RootValue allocRootValue(Value * v)
#endif
}
void Value::print(const SymbolTable &symbols, std::ostream &str,
std::set<const void *> *seen, int depth) const
void Value::print(const SymbolTable & symbols, std::ostream & str,
std::set<const void *> * seen) const
{
checkInterrupt();
if (depth <= 0) {
str << "«too deep»";
return;
}
switch (internalType) {
case tInt:
str << integer;
@@ -128,7 +123,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
str << "{ ";
for (auto & i : attrs->lexicographicOrder(symbols)) {
str << symbols[i->name] << " = ";
i->value->print(symbols, str, seen, depth - 1);
i->value->print(symbols, str, seen);
str << "; ";
}
str << "}";
@@ -144,7 +139,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
str << "[ ";
for (auto v2 : listItems()) {
if (v2)
v2->print(symbols, str, seen, depth - 1);
v2->print(symbols, str, seen);
else
str << "(nullptr)";
str << " ";
@@ -186,10 +181,11 @@ void Value::print(const SymbolTable &symbols, std::ostream &str,
}
}
void Value::print(const SymbolTable &symbols, std::ostream &str,
bool showRepeated, int depth) const {
void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const
{
std::set<const void *> seen;
print(symbols, str, showRepeated ? nullptr : &seen, depth);
print(symbols, str, showRepeated ? nullptr : &seen);
}
// Pretty print types for assertion errors
@@ -215,21 +211,20 @@ const Value * getPrimOp(const Value &v) {
return primOp;
}
std::string_view showType(ValueType type, bool withArticle)
std::string_view showType(ValueType type)
{
#define WA(a, w) withArticle ? a " " w : w
switch (type) {
case nInt: return WA("an", "integer");
case nBool: return WA("a", "Boolean");
case nString: return WA("a", "string");
case nPath: return WA("a", "path");
case nInt: return "an integer";
case nBool: return "a Boolean";
case nString: return "a string";
case nPath: return "a path";
case nNull: return "null";
case nAttrs: return WA("a", "set");
case nList: return WA("a", "list");
case nFunction: return WA("a", "function");
case nExternal: return WA("an", "external value");
case nFloat: return WA("a", "float");
case nThunk: return WA("a", "thunk");
case nAttrs: return "a set";
case nList: return "a list";
case nFunction: return "a function";
case nExternal: return "an external value";
case nFloat: return "a float";
case nThunk: return "a thunk";
}
abort();
}
@@ -498,7 +493,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr)
EvalState::EvalState(
const SearchPath & _searchPath,
const Strings & _searchPath,
ref<Store> store,
std::shared_ptr<Store> buildStore)
: sWith(symbols.create("<with>"))
@@ -563,32 +558,30 @@ EvalState::EvalState(
/* Initialise the Nix expression search path. */
if (!evalSettings.pureEval) {
for (auto & i : _searchPath.elements)
addToSearchPath(SearchPath::Elem {i});
for (auto & i : evalSettings.nixPath.get())
addToSearchPath(SearchPath::Elem::parse(i));
for (auto & i : _searchPath) addToSearchPath(i);
for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
}
if (evalSettings.restrictEval || evalSettings.pureEval) {
allowedPaths = PathSet();
for (auto & i : searchPath.elements) {
auto r = resolveSearchPathPath(i.path);
if (!r) continue;
for (auto & i : searchPath) {
auto r = resolveSearchPathElem(i);
if (!r.first) continue;
auto path = *std::move(r);
auto path = r.second;
if (store->isInStore(path)) {
if (store->isInStore(r.second)) {
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(path).first, closure);
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowPath(path);
} catch (InvalidPath &) {
allowPath(path);
allowPath(r.second);
}
} else
allowPath(path);
allowPath(r.second);
}
}
@@ -709,34 +702,28 @@ Path EvalState::toRealPath(const Path & path, const NixStringContext & context)
}
Value * EvalState::addConstant(const std::string & name, Value & v, Constant info)
Value * EvalState::addConstant(const std::string & name, Value & v)
{
Value * v2 = allocValue();
*v2 = v;
addConstant(name, v2, info);
addConstant(name, v2);
return v2;
}
void EvalState::addConstant(const std::string & name, Value * v, Constant info)
void EvalState::addConstant(const std::string & name, Value * v)
{
staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name;
baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
}
constantInfos.push_back({name2, info});
if (!(evalSettings.pureEval && info.impureOnly)) {
/* Check the type, if possible.
We might know the type of a thunk in advance, so be allowed
to just write it down in that case. */
if (auto gotType = v->type(true); gotType != nThunk)
assert(info.type == gotType);
/* Install value the base environment. */
staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
}
Value * EvalState::addPrimOp(const std::string & name,
size_t arity, PrimOpFun primOp)
{
return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name });
}
@@ -750,10 +737,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
vPrimOp->mkPrimOp(new PrimOp(primOp));
Value v;
v.mkApp(vPrimOp, vPrimOp);
return addConstant(primOp.name, v, {
.type = nThunk, // FIXME
.doc = primOp.doc,
});
return addConstant(primOp.name, v);
}
auto envName = symbols.create(primOp.name);
@@ -779,13 +763,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
auto v2 = &v;
if (auto * doc = v2->primOp->doc)
if (v2->primOp->doc)
return Doc {
.pos = {},
.name = v2->primOp->name,
.arity = v2->primOp->arity,
.args = v2->primOp->args,
.doc = doc,
.doc = v2->primOp->doc,
};
}
return {};
+24 -94
View File
@@ -9,7 +9,6 @@
#include "config.hh"
#include "experimental-features.hh"
#include "input-accessor.hh"
#include "search-path.hh"
#include <map>
#include <optional>
@@ -26,72 +25,15 @@ struct DerivedPath;
enum RepairFlag : bool;
/**
* Function that implements a primop.
*/
typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v);
/**
* Info about a primitive operation, and its implementation
*/
struct PrimOp
{
/**
* Name of the primop. `__` prefix is treated specially.
*/
std::string name;
/**
* Names of the parameters of a primop, for primops that take a
* fixed number of arguments to be substituted for these parameters.
*/
std::vector<std::string> args;
/**
* Aritiy of the primop.
*
* If `args` is not empty, this field will be computed from that
* field instead, so it doesn't need to be manually set.
*/
size_t arity = 0;
/**
* Optional free-form documentation about the primop.
*/
const char * doc = nullptr;
/**
* Implementation of the primop.
*/
PrimOpFun fun;
/**
* Optional experimental for this to be gated on.
*/
std::optional<ExperimentalFeature> experimentalFeature;
};
/**
* Info about a constant
*/
struct Constant
{
/**
* Optional type of the constant (known since it is a fixed value).
*
* @todo we should use an enum for this.
*/
ValueType type = nThunk;
/**
* Optional free-form documentation about the constant.
*/
size_t arity;
std::string name;
std::vector<std::string> args;
const char * doc = nullptr;
/**
* Whether the constant is impure, and not available in pure mode.
*/
bool impureOnly = false;
};
#if HAVE_BOEHMGC
@@ -123,6 +65,15 @@ std::string printValue(const EvalState & state, const Value & v);
std::ostream & operator << (std::ostream & os, const ValueType t);
struct SearchPathElem
{
std::string prefix;
// FIXME: maybe change this to an std::variant<SourcePath, URL>.
std::string path;
};
typedef std::list<SearchPathElem> SearchPath;
/**
* Initialise the Boehm GC, if applicable.
*/
@@ -309,7 +260,7 @@ private:
SearchPath searchPath;
std::map<std::string, std::optional<std::string>> searchPathResolved;
std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
/**
* Cache used by checkSourcePath().
@@ -336,12 +287,12 @@ private:
public:
EvalState(
const SearchPath & _searchPath,
const Strings & _searchPath,
ref<Store> store,
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
void addToSearchPath(SearchPath::Elem && elem);
void addToSearchPath(const std::string & s);
SearchPath getSearchPath() { return searchPath; }
@@ -423,16 +374,12 @@ public:
* Look up a file in the search path.
*/
SourcePath findFile(const std::string_view path);
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/**
* Try to resolve a search path value (not the optinal key part)
*
* If the specified search path element is a URI, download it.
*
* If it is not found, return `std::nullopt`
*/
std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path);
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
/**
* Evaluate an expression to normal form
@@ -566,23 +513,18 @@ public:
*/
std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
/**
* Name and documentation about every constant.
*
* Constants from primops are hard to crawl, and their docs will go
* here too.
*/
std::vector<std::pair<std::string, Constant>> constantInfos;
private:
unsigned int baseEnvDispl = 0;
void createBaseEnv();
Value * addConstant(const std::string & name, Value & v, Constant info);
Value * addConstant(const std::string & name, Value & v);
void addConstant(const std::string & name, Value * v, Constant info);
void addConstant(const std::string & name, Value * v);
Value * addPrimOp(const std::string & name,
size_t arity, PrimOpFun primOp);
Value * addPrimOp(PrimOp && primOp);
@@ -596,10 +538,6 @@ public:
std::optional<std::string> name;
size_t arity;
std::vector<std::string> args;
/**
* Unlike the other `doc` fields in this file, this one should never be
* `null`.
*/
const char * doc;
};
@@ -766,11 +704,8 @@ struct DebugTraceStacker {
/**
* @return A string representing the type of the value `v`.
*
* @param withArticle Whether to begin with an english article, e.g. "an
* integer" vs "integer".
*/
std::string_view showType(ValueType type, bool withArticle = true);
std::string_view showType(ValueType type);
std::string showType(const Value & v);
/**
@@ -802,12 +737,7 @@ struct EvalSettings : Config
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
R"(
List of directories to be searched for `<...>` file references
In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
)"};
"List of directories to be searched for `<...>` file references."};
Setting<bool> restrictEval{
this, false, "restrict-eval",
+3
View File
@@ -788,6 +788,9 @@ static RegisterPrimOp r2({
```nix
(builtins.getFlake "github:edolstra/dwarffs").rev
```
This function is only available if you enable the experimental feature
`flakes`.
)",
.fun = prim_getFlake,
.experimentalFeature = Xp::Flakes,
+1 -1
View File
@@ -36,7 +36,7 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
#define CUR_POS makeCurPos(*yylloc, data)
// backup to recover from yyless(0)
thread_local YYLTYPE prev_yylloc;
YYLTYPE prev_yylloc;
static void initLoc(YYLTYPE * loc)
{
+50 -44
View File
@@ -137,7 +137,6 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath,
dupAttr(state, ad.first, j2->second.pos, ad.second.pos);
jAttrs->attrs.emplace(ad.first, ad.second);
}
jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
} else {
dupAttr(state, attrPath, pos, j->second.pos);
}
@@ -276,12 +275,7 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols,
}
/* If this is a single string, then don't do a concatenation. */
if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
auto *const result = (*es2)[0].second;
delete es2;
return result;
}
return new ExprConcatStrings(pos, true, es2);
return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
}
@@ -664,7 +658,7 @@ Expr * EvalState::parse(
ParseData data {
.state = *this,
.symbols = symbols,
.basePath = basePath,
.basePath = std::move(basePath),
.origin = {origin},
};
@@ -735,9 +729,22 @@ Expr * EvalState::parseStdin()
}
void EvalState::addToSearchPath(SearchPath::Elem && elem)
void EvalState::addToSearchPath(const std::string & s)
{
searchPath.elements.emplace_back(std::move(elem));
size_t pos = s.find('=');
std::string prefix;
Path path;
if (pos == std::string::npos) {
path = s;
} else {
prefix = std::string(s, 0, pos);
path = std::string(s, pos + 1);
}
searchPath.emplace_back(SearchPathElem {
.prefix = prefix,
.path = path,
});
}
@@ -747,19 +754,22 @@ SourcePath EvalState::findFile(const std::string_view path)
}
SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos)
{
for (auto & i : searchPath.elements) {
auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
if (!suffixOpt) continue;
auto suffix = *suffixOpt;
auto rOpt = resolveSearchPathPath(i.path);
if (!rOpt) continue;
auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
for (auto & i : searchPath) {
std::string suffix;
if (i.prefix.empty())
suffix = concatStrings("/", path);
else {
auto s = i.prefix.size();
if (path.compare(0, s, i.prefix) != 0 ||
(path.size() > s && path[s] != '/'))
continue;
suffix = path.size() == s ? "" : concatStrings("/", path.substr(s));
}
auto r = resolveSearchPathElem(i);
if (!r.first) continue;
Path res = r.second + suffix;
if (pathExists(res)) return CanonPath(canonPath(res));
}
@@ -776,53 +786,49 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
}
std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem)
{
auto & value = value0.s;
auto i = searchPathResolved.find(value);
auto i = searchPathResolved.find(elem.path);
if (i != searchPathResolved.end()) return i->second;
std::optional<std::string> res;
std::pair<bool, std::string> res;
if (EvalSettings::isPseudoUrl(value)) {
if (EvalSettings::isPseudoUrl(elem.path)) {
try {
auto storePath = fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
res = { store->toRealPath(storePath) };
store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath;
res = { true, store->toRealPath(storePath) };
} catch (FileTransferError & e) {
logWarning({
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path)
});
res = std::nullopt;
res = { false, "" };
}
}
else if (hasPrefix(value, "flake:")) {
else if (hasPrefix(elem.path, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
debug("fetching flake search path element '%s''", value);
auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false);
debug("fetching flake search path element '%s''", elem.path);
auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
res = { store->toRealPath(storePath) };
res = { true, store->toRealPath(storePath) };
}
else {
auto path = absPath(value);
auto path = absPath(elem.path);
if (pathExists(path))
res = { path };
res = { true, path };
else {
logWarning({
.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path)
});
res = std::nullopt;
res = { false, "" };
}
}
if (res)
debug("resolved search path element '%s' to '%s'", value, *res);
else
debug("failed to resolve search path element '%s'", value);
debug("resolved search path element '%s' to '%s'", elem.path, res.second);
searchPathResolved[value] = res;
searchPathResolved[elem.path] = res;
return res;
}
+48 -260
View File
@@ -238,7 +238,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
}
}
static RegisterPrimOp primop_scopedImport(PrimOp {
static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info {
.name = "scopedImport",
.arity = 2,
.fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -692,7 +692,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a
v.listElems()[n++] = i;
}
static RegisterPrimOp primop_genericClosure(PrimOp {
static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
.name = "__genericClosure",
.args = {"attrset"},
.arity = 1,
@@ -809,7 +809,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
}
}
static RegisterPrimOp primop_addErrorContext(PrimOp {
static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info {
.name = "__addErrorContext",
.arity = 2,
.fun = prim_addErrorContext,
@@ -1300,10 +1300,9 @@ drvName, Bindings * attrs, Value & v)
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
DerivationOutput::CAFixed dof {
.ca = ContentAddress {
.method = std::move(method),
.hash = std::move(h),
},
.ca = ContentAddress::fromParts(
std::move(method),
std::move(h)),
};
drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
@@ -1401,7 +1400,7 @@ drvName, Bindings * attrs, Value & v)
v.mkAttrs(result);
}
static RegisterPrimOp primop_derivationStrict(PrimOp {
static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info {
.name = "derivationStrict",
.arity = 1,
.fun = prim_derivationStrict,
@@ -1503,8 +1502,6 @@ static RegisterPrimOp primop_storePath({
in a new path (e.g. `/nix/store/ld01dnzc…-source-source`).
Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
See also [`builtins.fetchClosure`](#builtins-fetchClosure).
)",
.fun = prim_storePath,
});
@@ -1659,9 +1656,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
}));
}
searchPath.elements.emplace_back(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = prefix },
.path = SearchPath::Path { .s = path },
searchPath.emplace_back(SearchPathElem {
.prefix = prefix,
.path = path,
});
}
@@ -1670,52 +1667,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos)));
}
static RegisterPrimOp primop_findFile(PrimOp {
static RegisterPrimOp primop_findFile(RegisterPrimOp::Info {
.name = "__findFile",
.args = {"search path", "lookup path"},
.doc = R"(
Look up the given path with the given search path.
A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`.
`prefix` is a relative path.
`path` denotes a file system location; the exact syntax depends on the command line interface.
Examples of search path attribute sets:
- ```
{
prefix = "nixos-config";
path = "/etc/nixos/configuration.nix";
}
```
- ```
{
prefix = "";
path = "/nix/var/nix/profiles/per-user/root/channels";
}
```
The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match.
This is the process for each entry:
If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`.
Note that the `path` may need to be downloaded at this point to look inside.
If the suffix is found inside that directory, then the entry is a match;
the combined absolute path of the directory (now downloaded if need be) and the suffix is returned.
The syntax
```nix
<nixpkgs>
```
is equivalent to:
```nix
builtins.findFile builtins.nixPath "nixpkgs"
```
)",
.arity = 2,
.fun = prim_findFile,
});
@@ -2165,8 +2119,10 @@ static void addPath(
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.method = method,
.hash = *expectedHash,
.hash = {
.method = method,
.hash = *expectedHash,
},
.references = {},
});
@@ -2432,7 +2388,7 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * *
state.mkPos(v, i->pos);
}
static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp {
static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info {
.name = "__unsafeGetAttrPos",
.arity = 2,
.fun = prim_unsafeGetAttrPos,
@@ -4105,10 +4061,10 @@ static RegisterPrimOp primop_splitVersion({
RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
RegisterPrimOp::RegisterPrimOp(PrimOp && primOp)
RegisterPrimOp::RegisterPrimOp(Info && info)
{
if (!primOps) primOps = new PrimOps;
primOps->push_back(std::move(primOp));
primOps->push_back(std::move(info));
}
@@ -4121,253 +4077,85 @@ void EvalState::createBaseEnv()
/* `builtins' must be first! */
v.mkAttrs(buildBindings(128).finish());
addConstant("builtins", v, {
.type = nAttrs,
.doc = R"(
Contains all the [built-in functions](@docroot@/language/builtins.md) and values.
Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
```nix
# if hasContext is not available, we assume `s` has a context
if builtins ? hasContext then builtins.hasContext s else true
```
)",
});
addConstant("builtins", v);
v.mkBool(true);
addConstant("true", v, {
.type = nBool,
.doc = R"(
Primitive value.
It can be returned by
[comparison operators](@docroot@/language/operators.md#Comparison)
and used in
[conditional expressions](@docroot@/language/constructs.md#Conditionals).
The name `true` is not special, and can be shadowed:
```nix-repl
nix-repl> let true = 1; in true
1
```
)",
});
addConstant("true", v);
v.mkBool(false);
addConstant("false", v, {
.type = nBool,
.doc = R"(
Primitive value.
It can be returned by
[comparison operators](@docroot@/language/operators.md#Comparison)
and used in
[conditional expressions](@docroot@/language/constructs.md#Conditionals).
The name `false` is not special, and can be shadowed:
```nix-repl
nix-repl> let false = 1; in false
1
```
)",
});
addConstant("false", v);
v.mkNull();
addConstant("null", v, {
.type = nNull,
.doc = R"(
Primitive value.
The name `null` is not special, and can be shadowed:
```nix-repl
nix-repl> let null = 1; in null
1
```
)",
});
addConstant("null", v);
if (!evalSettings.pureEval) {
v.mkInt(time(0));
}
addConstant("__currentTime", v, {
.type = nInt,
.doc = R"(
Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation.
Repeated references to that name will re-use the initially obtained value.
addConstant("__currentTime", v);
Example:
```console
$ nix repl
Welcome to Nix 2.15.1 Type :? for help.
nix-repl> builtins.currentTime
1683705525
nix-repl> builtins.currentTime
1683705525
```
The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
)",
.impureOnly = true,
});
if (!evalSettings.pureEval) {
v.mkString(settings.thisSystem.get());
addConstant("__currentSystem", v);
}
addConstant("__currentSystem", v, {
.type = nString,
.doc = R"(
The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval).
It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression:
```nix
builtins.derivation {
# ...
system = builtins.currentSystem;
}
```
It can be overridden in order to create derivations for different system than the current one:
```console
$ nix-instantiate --system "mips64-linux" --eval --expr 'builtins.currentSystem'
"mips64-linux"
```
)",
.impureOnly = true,
});
v.mkString(nixVersion);
addConstant("__nixVersion", v, {
.type = nString,
.doc = R"(
The version of Nix.
For example, where the command line returns the current Nix version,
```shell-session
$ nix --version
nix (Nix) 2.16.0
```
the Nix language evaluator returns the same value:
```nix-repl
nix-repl> builtins.nixVersion
"2.16.0"
```
)",
});
addConstant("__nixVersion", v);
v.mkString(store->storeDir);
addConstant("__storeDir", v, {
.type = nString,
.doc = R"(
Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use.
This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md):
```shell-session
$ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir
"/blah"
```
)",
});
addConstant("__storeDir", v);
/* Language version. This should be increased every time a new
language feature gets added. It's not necessary to increase it
when primops get added, because you can just use `builtins ?
primOp' to check. */
v.mkInt(6);
addConstant("__langVersion", v, {
.type = nInt,
.doc = R"(
The current version of the Nix language.
)",
});
addConstant("__langVersion", v);
// Miscellaneous
if (evalSettings.enableNativeCode) {
addPrimOp({
.name = "__importNative",
.arity = 2,
.fun = prim_importNative,
});
addPrimOp({
.name = "__exec",
.arity = 1,
.fun = prim_exec,
});
addPrimOp("__importNative", 2, prim_importNative);
addPrimOp("__exec", 1, prim_exec);
}
addPrimOp({
.fun = evalSettings.traceVerbose ? prim_trace : prim_second,
.arity = 2,
.name = "__traceVerbose",
.args = { "e1", "e2" },
.arity = 2,
.doc = R"(
Evaluate *e1* and print its abstract syntax representation on standard
error if `--trace-verbose` is enabled. Then return *e2*. This function
is useful for debugging.
)",
.fun = evalSettings.traceVerbose ? prim_trace : prim_second,
});
/* Add a value containing the current Nix expression search path. */
mkList(v, searchPath.elements.size());
mkList(v, searchPath.size());
int n = 0;
for (auto & i : searchPath.elements) {
for (auto & i : searchPath) {
auto attrs = buildBindings(2);
attrs.alloc("path").mkString(i.path.s);
attrs.alloc("prefix").mkString(i.prefix.s);
attrs.alloc("path").mkString(i.path);
attrs.alloc("prefix").mkString(i.prefix);
(v.listElems()[n++] = allocValue())->mkAttrs(attrs);
}
addConstant("__nixPath", v, {
.type = nList,
.doc = R"(
The search path used to resolve angle bracket path lookups.
Angle bracket expressions can be
[desugared](https://en.wikipedia.org/wiki/Syntactic_sugar)
using this and
[`builtins.findFile`](./builtins.html#builtins-findFile):
```nix
<nixpkgs>
```
is equivalent to:
```nix
builtins.findFile builtins.nixPath "nixpkgs"
```
)",
});
addConstant("__nixPath", v);
if (RegisterPrimOp::primOps)
for (auto & primOp : *RegisterPrimOp::primOps)
if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature))
if (!primOp.experimentalFeature
|| experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature))
{
auto primOpAdjusted = primOp;
primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity);
addPrimOp(std::move(primOpAdjusted));
addPrimOp({
.fun = primOp.fun,
.arity = std::max(primOp.args.size(), primOp.arity),
.name = primOp.name,
.args = primOp.args,
.doc = primOp.doc,
});
}
/* Add a wrapper around the derivation primop that computes the
`drvPath' and `outPath' attributes lazily.
Null docs because it is documented separately.
*/
`drvPath' and `outPath' attributes lazily. */
auto vDerivation = allocValue();
addConstant("derivation", vDerivation, {
.type = nFunction,
});
addConstant("derivation", vDerivation);
/* Now that we've added all primops, sort the `builtins' set,
because attribute lookups expect it to be sorted. */
+12 -2
View File
@@ -10,7 +10,17 @@ namespace nix {
struct RegisterPrimOp
{
typedef std::vector<PrimOp> PrimOps;
struct Info
{
std::string name;
std::vector<std::string> args;
size_t arity = 0;
const char * doc;
PrimOpFun fun;
std::optional<ExperimentalFeature> experimentalFeature;
};
typedef std::vector<Info> PrimOps;
static PrimOps * primOps;
/**
@@ -18,7 +28,7 @@ struct RegisterPrimOp
* will get called during EvalState initialization, so there
* may be primops not yet added and builtins is not yet sorted.
*/
RegisterPrimOp(PrimOp && primOp);
RegisterPrimOp(Info && info);
};
/* These primops are disabled without enableNativeCode, but plugins
+65 -186
View File
@@ -5,150 +5,37 @@
namespace nix {
/**
* Handler for the content addressed case.
*
* @param state Evaluator state and store to write to.
* @param fromStore Store containing the path to rewrite.
* @param fromPath Source path to be rewritten.
* @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path.
* @param v Return `Value`
*/
static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional<StorePath> & toPathMaybe, Value &v) {
// establish toPath or throw
if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) {
auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath);
if (toPathMaybe && *toPathMaybe != rewrittenPath)
throw Error({
.msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
state.store->printStorePath(fromPath),
state.store->printStorePath(rewrittenPath),
state.store->printStorePath(*toPathMaybe)),
.errPos = state.positions[pos]
});
if (!toPathMaybe)
throw Error({
.msg = hintfmt(
"rewriting '%s' to content-addressed form yielded '%s'\n"
"Use this value for the 'toPath' attribute passed to 'fetchClosure'",
state.store->printStorePath(fromPath),
state.store->printStorePath(rewrittenPath)),
.errPos = state.positions[pos]
});
}
auto toPath = *toPathMaybe;
// check and return
auto resultInfo = state.store->queryPathInfo(toPath);
if (!resultInfo->isContentAddressed(*state.store)) {
// We don't perform the rewriting when outPath already exists, as an optimisation.
// However, we can quickly detect a mistake if the toPath is input addressed.
throw Error({
.msg = hintfmt(
"The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n"
"Set 'toPath' to an empty string to make Nix report the correct content-addressed path.",
state.store->printStorePath(toPath)),
.errPos = state.positions[pos]
});
}
state.mkStorePathString(toPath, v);
}
/**
* Fetch the closure and make sure it's content addressed.
*/
static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) {
if (!state.store->isValidPath(fromPath))
copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath });
auto info = state.store->queryPathInfo(fromPath);
if (!info->isContentAddressed(*state.store)) {
throw Error({
.msg = hintfmt(
"The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n"
"If you do intend to fetch an input-addressed store path, add\n\n"
" inputAddressed = true;\n\n"
"to the 'fetchClosure' arguments.\n\n"
"Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.",
state.store->printStorePath(fromPath)),
.errPos = state.positions[pos]
});
}
state.mkStorePathString(fromPath, v);
}
/**
* Fetch the closure and make sure it's input addressed.
*/
static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) {
if (!state.store->isValidPath(fromPath))
copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath });
auto info = state.store->queryPathInfo(fromPath);
if (info->isContentAddressed(*state.store)) {
throw Error({
.msg = hintfmt(
"The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n"
"Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed",
state.store->printStorePath(fromPath)),
.errPos = state.positions[pos]
});
}
state.mkStorePathString(fromPath, v);
}
typedef std::optional<StorePath> StorePathOrGap;
static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure");
std::optional<std::string> fromStoreUrl;
std::optional<StorePath> fromPath;
std::optional<StorePathOrGap> toPath;
std::optional<bool> inputAddressedMaybe;
bool toCA = false;
std::optional<StorePath> toPath;
for (auto & attr : *args[0]->attrs) {
const auto & attrName = state.symbols[attr.name];
auto attrHint = [&]() -> std::string {
return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure";
};
if (attrName == "fromPath") {
NixStringContext context;
fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
fromPath = state.coerceToStorePath(attr.pos, *attr.value, context,
"while evaluating the 'fromPath' attribute passed to builtins.fetchClosure");
}
else if (attrName == "toPath") {
state.forceValue(*attr.value, attr.pos);
bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string("");
if (isEmptyString) {
toPath = StorePathOrGap {};
}
else {
toCA = true;
if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
NixStringContext context;
toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint());
toPath = state.coerceToStorePath(attr.pos, *attr.value, context,
"while evaluating the 'toPath' attribute passed to builtins.fetchClosure");
}
}
else if (attrName == "fromStore")
fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos,
attrHint());
else if (attrName == "inputAddressed")
inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint());
"while evaluating the 'fromStore' attribute passed to builtins.fetchClosure");
else
throw Error({
@@ -163,18 +50,6 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
.errPos = state.positions[pos]
});
bool inputAddressed = inputAddressedMaybe.value_or(false);
if (inputAddressed) {
if (toPath)
throw Error({
.msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them",
"inputAddressed",
"toPath"),
.errPos = state.positions[pos]
});
}
if (!fromStoreUrl)
throw Error({
.msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
@@ -199,40 +74,55 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg
auto fromStore = openStore(parsedURL.to_string());
if (toPath)
runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v);
else if (inputAddressed)
runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v);
else
runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v);
if (toCA) {
if (!toPath || !state.store->isValidPath(*toPath)) {
auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
auto i = remappings.find(*fromPath);
assert(i != remappings.end());
if (toPath && *toPath != i->second)
throw Error({
.msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
state.store->printStorePath(*fromPath),
state.store->printStorePath(i->second),
state.store->printStorePath(*toPath)),
.errPos = state.positions[pos]
});
if (!toPath)
throw Error({
.msg = hintfmt(
"rewriting '%s' to content-addressed form yielded '%s'; "
"please set this in the 'toPath' attribute passed to 'fetchClosure'",
state.store->printStorePath(*fromPath),
state.store->printStorePath(i->second)),
.errPos = state.positions[pos]
});
}
} else {
if (!state.store->isValidPath(*fromPath))
copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
toPath = fromPath;
}
/* In pure mode, require a CA path. */
if (evalSettings.pureEval) {
auto info = state.store->queryPathInfo(*toPath);
if (!info->isContentAddressed(*state.store))
throw Error({
.msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
state.store->printStorePath(*toPath)),
.errPos = state.positions[pos]
});
}
state.mkStorePathString(*toPath, v);
}
static RegisterPrimOp primop_fetchClosure({
.name = "__fetchClosure",
.args = {"args"},
.doc = R"(
Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context.
This function can be invoked in three ways, that we will discuss in order of preference.
**Fetch a content-addressed store path**
Example:
```nix
builtins.fetchClosure {
fromStore = "https://cache.nixos.org";
fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
}
```
This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations.
**Fetch any store path and rewrite it to a fully content-addressed store path**
Example:
Fetch a Nix store closure from a binary cache, rewriting it into
content-addressed form. For example,
```nix
builtins.fetchClosure {
@@ -242,42 +132,31 @@ static RegisterPrimOp primop_fetchClosure({
}
```
This example fetches `/nix/store/r2jd...` from the specified binary cache,
fetches `/nix/store/r2jd...` from the specified binary cache,
and rewrites it into the content-addressed store path
`/nix/store/ldbh...`.
Like the previous example, no extra configuration or privileges are required.
If `fromPath` is already content-addressed, or if you are
allowing impure evaluation (`--impure`), then `toPath` may be
omitted.
To find out the correct value for `toPath` given a `fromPath`,
use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md):
you can use `nix store make-content-addressed`:
```console
# nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
```
Alternatively, set `toPath = ""` and find the correct `toPath` in the error message.
This function is similar to `builtins.storePath` in that it
allows you to use a previously built store path in a Nix
expression. However, it is more reproducible because it requires
specifying a binary cache from which the path can be fetched.
Also, requiring a content-addressed final store path avoids the
need for users to configure binary cache public keys.
**Fetch an input-addressed store path as is**
Example:
```nix
builtins.fetchClosure {
fromStore = "https://cache.nixos.org";
fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
inputAddressed = true;
}
```
It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is.
However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration.
**`builtins.storePath`**
`fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression.
However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched.
Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
This function is only available if you enable the experimental
feature `fetch-closure`.
)",
.fun = prim_fetchClosure,
.experimentalFeature = Xp::FetchClosure,
+4 -2
View File
@@ -254,8 +254,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
auto expectedPath = state.store->makeFixedOutputPath(
name,
FixedOutputInfo {
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.hash = *expectedHash,
.hash = {
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.hash = *expectedHash,
},
.references = {}
});
-56
View File
@@ -1,56 +0,0 @@
#include "search-path.hh"
#include "util.hh"
namespace nix {
std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch(
std::string_view path) const
{
auto n = s.size();
/* Non-empty prefix and suffix must be separated by a /, or the
prefix is not a valid path prefix. */
bool needSeparator = n > 0 && (path.size() - n) > 0;
if (needSeparator && path[n] != '/') {
return std::nullopt;
}
/* Prefix must be prefix of this path. */
if (path.compare(0, n, s) != 0) {
return std::nullopt;
}
/* Skip next path separator. */
return {
path.substr(needSeparator ? n + 1 : n)
};
}
SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem)
{
size_t pos = rawElem.find('=');
return SearchPath::Elem {
.prefix = Prefix {
.s = pos == std::string::npos
? std::string { "" }
: std::string { rawElem.substr(0, pos) },
},
.path = Path {
.s = std::string { rawElem.substr(pos + 1) },
},
};
}
SearchPath parseSearchPath(const Strings & rawElems)
{
SearchPath res;
for (auto & rawElem : rawElems)
res.elements.emplace_back(SearchPath::Elem::parse(rawElem));
return res;
}
}
-108
View File
@@ -1,108 +0,0 @@
#pragma once
///@file
#include <optional>
#include "types.hh"
#include "comparator.hh"
namespace nix {
/**
* A "search path" is a list of ways look for something, used with
* `builtins.findFile` and `< >` lookup expressions.
*/
struct SearchPath
{
/**
* A single element of a `SearchPath`.
*
* Each element is tried in succession when looking up a path. The first
* element to completely match wins.
*/
struct Elem;
/**
* The first part of a `SearchPath::Elem` pair.
*
* Called a "prefix" because it takes the form of a prefix of a file
* path (first `n` path components). When looking up a path, to use
* a `SearchPath::Elem`, its `Prefix` must match the path.
*/
struct Prefix;
/**
* The second part of a `SearchPath::Elem` pair.
*
* It is either a path or a URL (with certain restrictions / extra
* structure).
*
* If the prefix of the path we are looking up matches, we then
* check if the rest of the path points to something that exists
* within the directory denoted by this. If so, the
* `SearchPath::Elem` as a whole matches, and that *something* being
* pointed to by the rest of the path we are looking up is the
* result.
*/
struct Path;
/**
* The list of search path elements. Each one is checked for a path
* when looking up. (The actual lookup entry point is in `EvalState`
* not in this class.)
*/
std::list<SearchPath::Elem> elements;
/**
* Parse a string into a `SearchPath`
*/
static SearchPath parse(const Strings & rawElems);
};
struct SearchPath::Prefix
{
/**
* Underlying string
*
* @todo Should we normalize this when constructing a `SearchPath::Prefix`?
*/
std::string s;
GENERATE_CMP(SearchPath::Prefix, me->s);
/**
* If the path possibly matches this search path element, return the
* suffix that we should look for inside the resolved value of the
* element
* Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist.
*/
std::optional<std::string_view> suffixIfPotentialMatch(std::string_view path) const;
};
struct SearchPath::Path
{
/**
* The location of a search path item, as a path or URL.
*
* @todo Maybe change this to `std::variant<SourcePath, URL>`.
*/
std::string s;
GENERATE_CMP(SearchPath::Path, me->s);
};
struct SearchPath::Elem
{
Prefix prefix;
Path path;
GENERATE_CMP(SearchPath::Elem, me->prefix, me->path);
/**
* Parse a string into a `SearchPath::Elem`
*/
static SearchPath::Elem parse(std::string_view rawElem);
};
}
-90
View File
@@ -1,90 +0,0 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "search-path.hh"
namespace nix {
TEST(SearchPathElem, parse_justPath) {
ASSERT_EQ(
SearchPath::Elem::parse("foo"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "" },
.path = SearchPath::Path { .s = "foo" },
}));
}
TEST(SearchPathElem, parse_emptyPrefix) {
ASSERT_EQ(
SearchPath::Elem::parse("=foo"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "" },
.path = SearchPath::Path { .s = "foo" },
}));
}
TEST(SearchPathElem, parse_oneEq) {
ASSERT_EQ(
SearchPath::Elem::parse("foo=bar"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "foo" },
.path = SearchPath::Path { .s = "bar" },
}));
}
TEST(SearchPathElem, parse_twoEqs) {
ASSERT_EQ(
SearchPath::Elem::parse("foo=bar=baz"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "foo" },
.path = SearchPath::Path { .s = "bar=baz" },
}));
}
TEST(SearchPathElem, suffixIfPotentialMatch_justPath) {
SearchPath::Prefix prefix { .s = "" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) {
SearchPath::Prefix prefix { .s = "fooX" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) {
SearchPath::Prefix prefix { .s = "foo/bar" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" });
}
}
-236
View File
@@ -1,236 +0,0 @@
#include "tests/libexpr.hh"
#include "value.hh"
namespace nix {
using namespace testing;
struct ValuePrintingTests : LibExprTest
{
template<class... A>
void test(Value v, std::string_view expected, A... args)
{
std::stringstream out;
v.print(state.symbols, out, args...);
ASSERT_EQ(out.str(), expected);
}
};
TEST_F(ValuePrintingTests, tInt)
{
Value vInt;
vInt.mkInt(10);
test(vInt, "10");
}
TEST_F(ValuePrintingTests, tBool)
{
Value vBool;
vBool.mkBool(true);
test(vBool, "true");
}
TEST_F(ValuePrintingTests, tString)
{
Value vString;
vString.mkString("some-string");
test(vString, "\"some-string\"");
}
TEST_F(ValuePrintingTests, tPath)
{
Value vPath;
vPath.mkString("/foo");
test(vPath, "\"/foo\"");
}
TEST_F(ValuePrintingTests, tNull)
{
Value vNull;
vNull.mkNull();
test(vNull, "null");
}
TEST_F(ValuePrintingTests, tAttrs)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
BindingsBuilder builder(state, state.allocBindings(10));
builder.insert(state.symbols.create("one"), &vOne);
builder.insert(state.symbols.create("two"), &vTwo);
Value vAttrs;
vAttrs.mkAttrs(builder.finish());
test(vAttrs, "{ one = 1; two = 2; }");
}
TEST_F(ValuePrintingTests, tList)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
Value vList;
state.mkList(vList, 5);
vList.bigList.elems[0] = &vOne;
vList.bigList.elems[1] = &vTwo;
vList.bigList.size = 3;
test(vList, "[ 1 2 (nullptr) ]");
}
TEST_F(ValuePrintingTests, vThunk)
{
Value vThunk;
vThunk.mkThunk(nullptr, nullptr);
test(vThunk, "<CODE>");
}
TEST_F(ValuePrintingTests, vApp)
{
Value vApp;
vApp.mkApp(nullptr, nullptr);
test(vApp, "<CODE>");
}
TEST_F(ValuePrintingTests, vLambda)
{
Value vLambda;
vLambda.mkLambda(nullptr, nullptr);
test(vLambda, "<LAMBDA>");
}
TEST_F(ValuePrintingTests, vPrimOp)
{
Value vPrimOp;
vPrimOp.mkPrimOp(nullptr);
test(vPrimOp, "<PRIMOP>");
}
TEST_F(ValuePrintingTests, vPrimOpApp)
{
Value vPrimOpApp;
vPrimOpApp.mkPrimOpApp(nullptr, nullptr);
test(vPrimOpApp, "<PRIMOP-APP>");
}
TEST_F(ValuePrintingTests, vExternal)
{
struct MyExternal : ExternalValueBase
{
public:
std::string showType() const override
{
return "";
}
std::string typeOf() const override
{
return "";
}
virtual std::ostream & print(std::ostream & str) const override
{
str << "testing-external!";
return str;
}
} myExternal;
Value vExternal;
vExternal.mkExternal(&myExternal);
test(vExternal, "testing-external!");
}
TEST_F(ValuePrintingTests, vFloat)
{
Value vFloat;
vFloat.mkFloat(2.0);
test(vFloat, "2");
}
TEST_F(ValuePrintingTests, vBlackhole)
{
Value vBlackhole;
vBlackhole.mkBlackhole();
test(vBlackhole, "«potential infinite recursion»");
}
TEST_F(ValuePrintingTests, depthAttrs)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
BindingsBuilder builder(state, state.allocBindings(10));
builder.insert(state.symbols.create("one"), &vOne);
builder.insert(state.symbols.create("two"), &vTwo);
Value vAttrs;
vAttrs.mkAttrs(builder.finish());
BindingsBuilder builder2(state, state.allocBindings(10));
builder2.insert(state.symbols.create("one"), &vOne);
builder2.insert(state.symbols.create("two"), &vTwo);
builder2.insert(state.symbols.create("nested"), &vAttrs);
Value vNested;
vNested.mkAttrs(builder2.finish());
test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1);
test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2);
test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3);
test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4);
}
TEST_F(ValuePrintingTests, depthList)
{
Value vOne;
vOne.mkInt(1);
Value vTwo;
vTwo.mkInt(2);
BindingsBuilder builder(state, state.allocBindings(10));
builder.insert(state.symbols.create("one"), &vOne);
builder.insert(state.symbols.create("two"), &vTwo);
Value vAttrs;
vAttrs.mkAttrs(builder.finish());
BindingsBuilder builder2(state, state.allocBindings(10));
builder2.insert(state.symbols.create("one"), &vOne);
builder2.insert(state.symbols.create("two"), &vTwo);
builder2.insert(state.symbols.create("nested"), &vAttrs);
Value vNested;
vNested.mkAttrs(builder2.finish());
Value vList;
state.mkList(vList, 5);
vList.bigList.elems[0] = &vOne;
vList.bigList.elems[1] = &vTwo;
vList.bigList.elems[2] = &vNested;
vList.bigList.size = 3;
test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1);
test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2);
test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3);
test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4);
test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5);
}
} // namespace nix
+4 -11
View File
@@ -2,7 +2,6 @@
///@file
#include <cassert>
#include <climits>
#include "symbol-table.hh"
#include "value/context.hh"
@@ -138,11 +137,11 @@ private:
friend std::string showType(const Value & v);
void print(const SymbolTable &symbols, std::ostream &str, std::set<const void *> *seen, int depth) const;
void print(const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen) const;
public:
void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const;
void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const;
// Functions needed to distinguish the type
// These should be removed eventually, by putting the functionality that's
@@ -219,11 +218,8 @@ public:
/**
* Returns the normal type of a Value. This only returns nThunk if
* the Value hasn't been forceValue'd
*
* @param invalidIsThunk Instead of aborting an an invalid (probably
* 0, so uninitialized) internal type, return `nThunk`.
*/
inline ValueType type(bool invalidIsThunk = false) const
inline ValueType type() const
{
switch (internalType) {
case tInt: return nInt;
@@ -238,10 +234,7 @@ public:
case tFloat: return nFloat;
case tThunk: case tApp: case tBlackhole: return nThunk;
}
if (invalidIsThunk)
return nThunk;
else
abort();
abort();
}
/**
+4 -2
View File
@@ -217,8 +217,10 @@ StorePath Input::computeStorePath(Store & store) const
if (!narHash)
throw Error("cannot compute store path for unlocked input '%s'", to_string());
return store.makeFixedOutputPath(getName(), FixedOutputInfo {
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
.references = {},
});
}
+4 -2
View File
@@ -77,8 +77,10 @@ DownloadFileResult downloadFile(
*store,
name,
FixedOutputInfo {
.method = FileIngestionMethod::Flat,
.hash = hash,
.hash = {
.method = FileIngestionMethod::Flat,
.hash = hash,
},
.references = {},
},
hashString(htSHA256, sink.s),
+10 -6
View File
@@ -309,8 +309,10 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
*this,
name,
FixedOutputInfo {
.method = method,
.hash = nar.first,
.hash = {
.method = method,
.hash = nar.first,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -426,8 +428,10 @@ StorePath BinaryCacheStore::addToStore(
*this,
name,
FixedOutputInfo {
.method = method,
.hash = h,
.hash = {
.method = method,
.hash = h,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -461,8 +465,8 @@ StorePath BinaryCacheStore::addTextToStore(
*this,
std::string { name },
TextInfo {
.hash = textHash,
.references = references,
{ .hash = textHash },
references,
},
nar.first,
};
+17 -22
View File
@@ -1,5 +1,5 @@
#include "local-derivation-goal.hh"
#include "indirect-root-store.hh"
#include "gc-store.hh"
#include "hook-instance.hh"
#include "worker.hh"
#include "builtins.hh"
@@ -395,9 +395,8 @@ static void linkOrCopy(const Path & from, const Path & to)
bind-mount in this case?
It can also fail with EPERM in BeegFS v7 and earlier versions
or fail with EXDEV in OpenAFS
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM && errno != EXDEV)
if (errno != EMLINK && errno != EPERM)
throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to);
}
@@ -594,10 +593,6 @@ void LocalDerivationGoal::startBuilder()
else
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
}
if (hasPrefix(worker.store.storeDir, tmpDirInSandbox))
{
throw Error("`sandbox-build-dir` must not contain the storeDir");
}
dirsInChroot[tmpDirInSandbox] = tmpDir;
/* Add the closure of store paths to the chroot. */
@@ -912,13 +907,15 @@ void LocalDerivationGoal::startBuilder()
openSlave();
/* Drop additional groups here because we can't do it
after we've created the new user namespace. */
if (setgroups(0, 0) == -1) {
if (errno != EPERM)
throw SysError("setgroups failed");
if (settings.requireDropSupplementaryGroups)
throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step.");
}
after we've created the new user namespace. FIXME:
this means that if we're not root in the parent
namespace, we can't drop additional groups; they will
be mapped to nogroup in the child namespace. There does
not seem to be a workaround for this. (But who can tell
from reading user_namespaces(7)?)
See also https://lwn.net/Articles/621612/. */
if (getuid() == 0 && setgroups(0, 0) == -1)
throw SysError("setgroups failed");
ProcessOptions options;
options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
@@ -1200,7 +1197,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
/* A wrapper around LocalStore that only allows building/querying of
paths that are in the input closures of the build or were added via
recursive Nix calls. */
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
{
ref<LocalStore> next;
@@ -1251,13 +1248,11 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
void queryReferrers(const StorePath & path, StorePathSet & referrers) override
{ }
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore = nullptr) override
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
{
if (!goal.isAllowed(path))
throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
return next->queryPartialDerivationOutputMap(path, evalStore);
return next->queryPartialDerivationOutputMap(path);
}
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
@@ -2542,16 +2537,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
},
[&](const DerivationOutput::CAFixed & dof) {
auto & wanted = dof.ca.hash;
auto wanted = dof.ca.getHash();
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.ca.method,
.method = dof.ca.getMethod(),
.hashType = wanted.type,
});
/* Check wanted hash */
assert(newInfo0.ca);
auto & got = newInfo0.ca->hash;
auto got = newInfo0.ca->getHash();
if (wanted != got) {
/* Throw an error after registering the path as
valid. */
+92 -33
View File
@@ -4,6 +4,11 @@
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const
{
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(FileIngestionMethod m)
{
switch (m) {
@@ -37,6 +42,21 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
return method;
}
std::string ContentAddress::render() const
{
return std::visit(overloaded {
[](const TextHash & th) {
return "text:"
+ th.hash.to_string(Base32, true);
},
[](const FixedOutputHash & fsh) {
return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
}
}, raw);
}
std::string ContentAddressMethod::render(HashType ht) const
{
return std::visit(overloaded {
@@ -49,20 +69,6 @@ std::string ContentAddressMethod::render(HashType ht) const
}, raw);
}
std::string ContentAddress::render() const
{
return std::visit(overloaded {
[](const TextIngestionMethod &) -> std::string {
return "text:";
},
[](const FileIngestionMethod & method) {
return "fixed:"
+ makeFileIngestionPrefix(method);
},
}, method.raw)
+ this->hash.to_string(Base32, true);
}
/**
* Parses content address strings up to the hash.
*/
@@ -112,12 +118,22 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
{
auto rest = rawCa;
auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
auto hashType = hashType_; // work around clang bug
return ContentAddress {
.method = std::move(caMethod).raw,
.hash = Hash::parseNonSRIUnprefixed(rest, hashType),
};
return std::visit(overloaded {
[&](TextIngestionMethod &) {
return ContentAddress(TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, hashType)
});
},
[&](FileIngestionMethod & fim) {
return ContentAddress(FixedOutputHash {
.method = fim,
.hash = Hash::parseNonSRIUnprefixed(rest, hashType),
});
},
}, caMethod.raw);
}
std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
@@ -140,10 +156,52 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
return ca ? ca->render() : "";
}
ContentAddress ContentAddress::fromParts(
ContentAddressMethod method, Hash hash) noexcept
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> ContentAddress {
return TextHash {
.hash = std::move(hash),
};
},
[&](FileIngestionMethod m2) -> ContentAddress {
return FixedOutputHash {
.method = std::move(m2),
.hash = std::move(hash),
};
},
}, method.raw);
}
ContentAddressMethod ContentAddress::getMethod() const
{
return std::visit(overloaded {
[](const TextHash & th) -> ContentAddressMethod {
return TextIngestionMethod {};
},
[](const FixedOutputHash & fsh) -> ContentAddressMethod {
return fsh.method;
},
}, raw);
}
const Hash & ContentAddress::getHash() const
{
return std::visit(overloaded {
[](const TextHash & th) -> auto & {
return th.hash;
},
[](const FixedOutputHash & fsh) -> auto & {
return fsh.hash;
},
}, raw);
}
std::string ContentAddress::printMethodAlgo() const
{
return method.renderPrefix()
+ printHashType(hash.type);
return getMethod().renderPrefix()
+ printHashType(getHash().type);
}
bool StoreReferences::empty() const
@@ -159,20 +217,19 @@ size_t StoreReferences::size() const
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
{
return std::visit(overloaded {
[&](const TextIngestionMethod &) -> ContentAddressWithReferences {
[&](const TextHash & h) -> ContentAddressWithReferences {
return TextInfo {
.hash = ca.hash,
.hash = h,
.references = {},
};
},
[&](const FileIngestionMethod & method) -> ContentAddressWithReferences {
[&](const FixedOutputHash & h) -> ContentAddressWithReferences {
return FixedOutputInfo {
.method = method,
.hash = ca.hash,
.hash = h,
.references = {},
};
},
}, ca.method.raw);
}, ca.raw);
}
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
@@ -184,7 +241,7 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
return std::nullopt;
return ContentAddressWithReferences {
TextInfo {
.hash = std::move(hash),
.hash = { .hash = std::move(hash) },
.references = std::move(refs.others),
}
};
@@ -192,8 +249,10 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
return ContentAddressWithReferences {
FixedOutputInfo {
.method = m2,
.hash = std::move(hash),
.hash = {
.method = m2,
.hash = std::move(hash),
},
.references = std::move(refs),
}
};
@@ -208,7 +267,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const
return TextIngestionMethod {};
},
[](const FixedOutputInfo & fsh) -> ContentAddressMethod {
return fsh.method;
return fsh.hash.method;
},
}, raw);
}
@@ -217,10 +276,10 @@ Hash ContentAddressWithReferences::getHash() const
{
return std::visit(overloaded {
[](const TextInfo & th) {
return th.hash;
return th.hash.hash;
},
[](const FixedOutputInfo & fsh) {
return fsh.hash;
return fsh.hash.hash;
},
}, raw);
}
+61 -30
View File
@@ -113,6 +113,37 @@ struct ContentAddressMethod
* Mini content address
*/
/**
* Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently.
*/
struct TextHash {
/**
* Hash of the contents of the text/file.
*/
Hash hash;
GENERATE_CMP(TextHash, me->hash);
};
/**
* Used by most store objects that are content-addressed.
*/
struct FixedOutputHash {
/**
* How the file system objects are serialized
*/
FileIngestionMethod method;
/**
* Hash of that serialization
*/
Hash hash;
std::string printMethodAlgo() const;
GENERATE_CMP(FixedOutputHash, me->method, me->hash);
};
/**
* We've accumulated several types of content-addressed paths over the
* years; fixed-output derivations support multiple hash algorithms and
@@ -127,17 +158,19 @@ struct ContentAddressMethod
*/
struct ContentAddress
{
/**
* How the file system objects are serialized
*/
ContentAddressMethod method;
typedef std::variant<
TextHash,
FixedOutputHash
> Raw;
/**
* Hash of that serialization
*/
Hash hash;
Raw raw;
GENERATE_CMP(ContentAddress, me->method, me->hash);
GENERATE_CMP(ContentAddress, me->raw);
/* The moral equivalent of `using Raw::Raw;` */
ContentAddress(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
/**
* Compute the content-addressability assertion
@@ -150,6 +183,20 @@ struct ContentAddress
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
/**
* Create a `ContentAddress` from 2 parts:
*
* @param method Way ingesting the file system data.
*
* @param hash Hash of ingested file system data.
*/
static ContentAddress fromParts(
ContentAddressMethod method, Hash hash) noexcept;
ContentAddressMethod getMethod() const;
const Hash & getHash() const;
std::string printMethodAlgo() const;
};
@@ -172,8 +219,7 @@ std::string renderContentAddress(std::optional<ContentAddress> ca);
* References to other store objects are tracked with store paths, self
* references however are tracked with a boolean.
*/
struct StoreReferences
{
struct StoreReferences {
/**
* References to other store objects
*/
@@ -200,13 +246,8 @@ struct StoreReferences
};
// This matches the additional info that we need for makeTextPath
struct TextInfo
{
/**
* Hash of the contents of the text/file.
*/
Hash hash;
struct TextInfo {
TextHash hash;
/**
* References to other store objects only; self references
* disallowed
@@ -216,18 +257,8 @@ struct TextInfo
GENERATE_CMP(TextInfo, me->hash, me->references);
};
struct FixedOutputInfo
{
/**
* How the file system objects are serialized
*/
FileIngestionMethod method;
/**
* Hash of that serialization
*/
Hash hash;
struct FixedOutputInfo {
FixedOutputHash hash;
/**
* References to other store objects or this one.
*/
+4 -3
View File
@@ -7,7 +7,6 @@
#include "store-cast.hh"
#include "gc-store.hh"
#include "log-store.hh"
#include "indirect-root-store.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "archive.hh"
@@ -676,8 +675,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
Path path = absPath(readString(from));
logger->startWork();
auto & indirectRootStore = require<IndirectRootStore>(*store);
indirectRootStore.addIndirectRoot(path);
auto & gcStore = require<GcStore>(*store);
gcStore.addIndirectRoot(path);
logger->stopWork();
to << 1;
@@ -865,6 +864,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
StringSet sigs = readStrings<StringSet>(from);
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add signatures");
store->addSignatures(path, sigs);
logger->stopWork();
to << 1;
+10 -12
View File
@@ -232,10 +232,9 @@ static DerivationOutput parseDerivationOutput(const Store & store,
validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
return DerivationOutput::CAFixed {
.ca = ContentAddress {
.method = std::move(method),
.hash = std::move(hash),
},
.ca = ContentAddress::fromParts(
std::move(method),
std::move(hash)),
};
} else {
experimentalFeatureSettings.require(Xp::CaDerivations);
@@ -396,7 +395,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false));
s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
},
[&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
@@ -629,7 +628,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.ca.printMethodAlgo() + ":"
+ dof.ca.hash.to_string(Base16, false) + ":"
+ dof.ca.getHash().to_string(Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
@@ -781,7 +780,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
[&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.ca.printMethodAlgo()
<< dof.ca.hash.to_string(Base16, false);
<< dof.ca.getHash().to_string(Base16, false);
},
[&](const DerivationOutput::CAFloating & dof) {
out << ""
@@ -971,7 +970,7 @@ nlohmann::json DerivationOutput::toJSON(
[&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.ca.printMethodAlgo();
res["hash"] = dof.ca.hash.to_string(Base16, false);
res["hash"] = dof.ca.getHash().to_string(Base16, false);
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
@@ -1018,10 +1017,9 @@ DerivationOutput DerivationOutput::fromJSON(
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
auto [method, hashType] = methodAlgo();
auto dof = DerivationOutput::CAFixed {
.ca = ContentAddress {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
},
.ca = ContentAddress::fromParts(
std::move(method),
Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
};
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
throw Error("Path doesn't match derivation output");
+56
View File
@@ -1,4 +1,5 @@
#include "derived-path.hh"
#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
@@ -29,6 +30,30 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
return res;
}
nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = store->printStorePath(path);
}
return res;
}
StorePathSet BuiltPath::outPaths() const
{
return std::visit(
overloaded{
[](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
[](const BuiltPath::Built & b) {
StorePathSet res;
for (auto & [_, path] : b.outputs)
res.insert(path);
return res;
},
}, raw()
);
}
std::string DerivedPath::Opaque::to_string(const Store & store) const
{
return store.printStorePath(path);
@@ -96,4 +121,35 @@ DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
return parseWith(store, s, "!");
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
{
RealisedPath::Set res;
std::visit(
overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath));
for (auto& [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled(
Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath), outputName);
auto thisRealisation = store.queryRealisation(
DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // We’ve built it, so we must
// have the realisation
res.insert(*thisRealisation);
} else {
res.insert(outputPath);
}
}
},
},
raw());
return res;
}
}
+42
View File
@@ -3,6 +3,7 @@
#include "util.hh"
#include "path.hh"
#include "realisation.hh"
#include "outputs-spec.hh"
#include "comparator.hh"
@@ -108,6 +109,47 @@ struct DerivedPath : _DerivedPathRaw {
static DerivedPath parseLegacy(const Store & store, std::string_view);
};
/**
* A built derived path with hints in the form of optional concrete output paths.
*
* See 'BuiltPath' for more an explanation.
*/
struct BuiltPathBuilt {
StorePath drvPath;
std::map<std::string, StorePath> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static BuiltPathBuilt parse(const Store & store, std::string_view);
GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs);
};
using _BuiltPathRaw = std::variant<
DerivedPath::Opaque,
BuiltPathBuilt
>;
/**
* A built path. Similar to a DerivedPath, but enriched with the corresponding
* output path(s).
*/
struct BuiltPath : _BuiltPathRaw {
using Raw = _BuiltPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = BuiltPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
StorePathSet outPaths() const;
RealisedPath::Set toRealisedPaths(Store & store) const;
};
typedef std::vector<DerivedPath> DerivedPaths;
typedef std::vector<BuiltPath> BuiltPaths;
}
+9 -26
View File
@@ -71,36 +71,19 @@ struct GCResults
};
/**
* Mix-in class for \ref Store "stores" which expose a notion of garbage
* collection.
*
* Garbage collection will allow deleting paths which are not
* transitively "rooted".
*
* The notion of GC roots actually not part of this class.
*
* - The base `Store` class has `Store::addTempRoot()` because for a store
* that doesn't support garbage collection at all, a temporary GC root is
* safely implementable as no-op.
*
* @todo actually this is not so good because stores are *views*.
* Some views have only a no-op temp roots even though others to the
* same store allow triggering GC. For instance one can't add a root
* over ssh, but that doesn't prevent someone from gc-ing that store
* accesed via SSH locally).
*
* - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`,
* which is not part of this class because it relies on the notion of
* an ambient file system. There are stores (`ssh-ng://`, for one),
* that *do* support garbage collection but *don't* expose any file
* system, and `LocalFSStore::addPermRoot` thus does not make sense
* for them.
*/
struct GcStore : public virtual Store
{
inline static std::string operationName = "Garbage collection";
/**
* Add an indirect root, which is merely a symlink to `path` from
* `/nix/var/nix/gcroots/auto/<hash of path>`. `path` is supposed
* to be a symlink to a store path. The garbage collector will
* automatically remove the indirect root when it finds that
* `path` has disappeared.
*/
virtual void addIndirectRoot(const Path & path) = 0;
/**
* Find the roots of the garbage collector. Each root is a pair
* `(link, storepath)` where `link` is the path of the symlink
+2 -1
View File
@@ -1,6 +1,7 @@
#include "derivations.hh"
#include "globals.hh"
#include "local-store.hh"
#include "local-fs-store.hh"
#include "finally.hh"
#include <functional>
@@ -49,7 +50,7 @@ void LocalStore::addIndirectRoot(const Path & path)
}
Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
{
Path gcRoot(canonPath(_gcRoot));
+24 -50
View File
@@ -193,24 +193,18 @@ public:
Setting<std::string> thisSystem{
this, SYSTEM, "system",
R"(
The system type of the current Nix installation.
Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
This option specifies the canonical Nix system name of the current
installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
build derivations whose `system` attribute equals the value
specified here. In general, it never makes sense to modify this
value from its default, since you can use it to ‘lie’ about the
platform you are building on (e.g., perform a Mac OS build on a
Linux machine; the result would obviously be wrong). It only makes
sense if the Nix binaries can run on multiple platforms, e.g.,
‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
The default value is set when Nix itself is compiled for the system it will run on.
The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms):
- `x86_64-linux`
- `x86_64-darwin`
- `i686-linux`
- `aarch64-linux`
- `aarch64-darwin`
- `armv6l-linux`
- `armv7l-linux`
In general, you do not have to modify this setting.
While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong.
This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem).
It defaults to the canonical Nix system name detected by `configure`
at build time.
)"};
Setting<time_t> maxSilentTime{
@@ -530,24 +524,6 @@ public:
Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."};
Setting<bool> requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups",
R"(
Following the principle of least privilege,
Nix will attempt to drop supplementary groups when building with sandboxing.
However this can fail under some circumstances.
For example, if the user lacks the `CAP_SETGID` capability.
Search `setgroups(2)` for `EPERM` to find more detailed information on this.
If you encounter such a failure, setting this option to `false` will let you ignore it and continue.
But before doing so, you should consider the security implications carefully.
Not dropping supplementary groups means the build sandbox will be less restricted than intended.
This option defaults to `true` when the user is root
(since `root` usually has permissions to call setgroups)
and `false` otherwise.
)"};
#if __linux__
Setting<std::string> sandboxShmSize{
this, "50%", "sandbox-dev-shm-size",
@@ -676,20 +652,18 @@ public:
getDefaultExtraPlatforms(),
"extra-platforms",
R"(
System types of executables that can be run on this machine.
Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system).
Setting this can be useful to build derivations locally on compatible machines:
- `i686-linux` executables can be run on `x86_64-linux` machines (set by default)
- `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable)
- `armv6` and `armv5tel` executables can be run on `armv7`
- some `aarch64` machines can also natively run 32-bit ARM code
- `qemu-user` may be used to support non-native platforms (though this
may be slow and buggy)
Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation.
You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation.
Platforms other than the native one which this machine is capable of
building for. This can be useful for supporting additional
architectures on compatible machines: i686-linux can be built on
x86\_64-linux machines (and the default for this setting reflects
this); armv7 is backwards-compatible with armv6 and armv5tel; some
aarch64 machines can also natively run 32-bit ARM code; and
qemu-user may be used to support non-native platforms (though this
may be slow and buggy). Most values for this are not enabled by
default because build systems will often misdetect the target
platform and generate incompatible code, so you may wish to
cross-check the results of using this option against proper
natively-built versions of your derivations.
)", {}, false};
Setting<StringSet> systemFeatures{
@@ -1018,7 +992,7 @@ public:
| `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` |
| `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` |
If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option.
If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option.
If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`.
This can be achieved with the following shell commands:
-48
View File
@@ -1,48 +0,0 @@
#pragma once
///@file
#include "local-fs-store.hh"
namespace nix {
/**
* Mix-in class for implementing permanent roots as a pair of a direct
* (strong) reference and indirect weak reference to the first
* reference.
*
* See methods for details on the operations it represents.
*/
struct IndirectRootStore : public virtual LocalFSStore
{
inline static std::string operationName = "Indirect GC roots registration";
/**
* Implementation of `LocalFSStore::addPermRoot` where the permanent
* root is a pair of
*
* - The user-facing symlink which all implementations must create
*
* - An additional weak reference known as the "indirect root" that
* points to that symlink.
*
* The garbage collector will automatically remove the indirect root
* when it finds that the symlink has disappeared.
*
* The implementation of this method is concrete, but it delegates
* to `addIndirectRoot()` which is abstract.
*/
Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final;
/**
* Add an indirect root, which is a weak reference to the
* user-facing symlink created by `addPermRoot()`.
*
* @param path user-facing and user-controlled symlink to a store
* path.
*
* The form this weak-reference takes is implementation-specific.
*/
virtual void addIndirectRoot(const Path & path) = 0;
};
}
+2 -14
View File
@@ -40,7 +40,6 @@ class LocalFSStore : public virtual LocalFSStoreConfig,
public virtual LogStore
{
public:
inline static std::string operationName = "Local Filesystem Store";
const static std::string drvsLogDir;
@@ -50,20 +49,9 @@ public:
ref<FSAccessor> getFSAccessor() override;
/**
* Creates symlink from the `gcRoot` to the `storePath` and
* registers the `gcRoot` as a permanent GC root. The `gcRoot`
* symlink lives outside the store and is created and owned by the
* user.
*
* @param gcRoot The location of the symlink.
*
* @param storePath The store object being rooted. The symlink will
* point to `toRealPath(store.printStorePath(storePath))`.
*
* How the permanent GC root corresponding to this symlink is
* managed is implementation-specific.
* Register a permanent GC root.
*/
virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0;
Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
virtual Path getRealStoreDir() { return realStoreDir; }
+61 -42
View File
@@ -1022,9 +1022,10 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
std::map<std::string, std::optional<StorePath>>
LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path)
LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
{
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto path = path_;
auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock());
std::map<std::string, std::optional<StorePath>> outputs;
uint64_t drvId;
@@ -1036,6 +1037,21 @@ LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path)
return outputs;
});
if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
return outputs;
auto drv = readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto& [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation)
outputs.insert_or_assign(outputName, realisation->outPath);
else
outputs.insert({outputName, std::nullopt});
}
return outputs;
}
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
@@ -1233,17 +1249,27 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
printStorePath(info.path), info.narSize, hashResult.second);
if (info.ca) {
auto & specified = *info.ca;
auto actualHash = hashCAPath(
specified.method,
specified.hash.type,
info.path
);
if (specified.hash != actualHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
specified.hash.to_string(Base32, true),
actualHash.hash.to_string(Base32, true));
if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) {
auto actualFoHash = hashCAPath(
foHash->method,
foHash->hash.type,
info.path
);
if (foHash->hash != actualFoHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
foHash->hash.to_string(Base32, true),
actualFoHash.hash.to_string(Base32, true));
}
}
if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) {
auto actualTextHash = hashString(htSHA256, readFile(realPath));
if (textHash->hash != actualTextHash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
textHash->hash.to_string(Base32, true),
actualTextHash.to_string(Base32, true));
}
}
}
@@ -1323,8 +1349,10 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
auto [hash, size] = hashSink->finish();
ContentAddressWithReferences desc = FixedOutputInfo {
.method = method,
.hash = hash,
.hash = {
.method = method,
.hash = hash,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1400,8 +1428,8 @@ StorePath LocalStore::addTextToStore(
{
auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, TextInfo {
.hash = hash,
.references = references,
{ .hash = hash },
references,
});
addTempRoot(dstPath);
@@ -1431,10 +1459,7 @@ StorePath LocalStore::addTextToStore(
ValidPathInfo info { dstPath, narHash };
info.narSize = sink.s.size();
info.references = references;
info.ca = {
.method = TextIngestionMethod {},
.hash = hash,
};
info.ca = TextHash { .hash = hash };
registerValidPath(info);
}
@@ -1831,39 +1856,33 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
}
}
ContentAddress LocalStore::hashCAPath(
const ContentAddressMethod & method, const HashType & hashType,
FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method, const HashType & hashType,
const StorePath & path)
{
return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart());
}
ContentAddress LocalStore::hashCAPath(
const ContentAddressMethod & method,
FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method,
const HashType & hashType,
const Path & path,
const std::string_view pathHash
)
{
HashModuloSink caSink ( hashType, std::string(pathHash) );
std::visit(overloaded {
[&](const TextIngestionMethod &) {
readFile(path, caSink);
},
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(path, caSink);
break;
case FileIngestionMethod::Flat:
readFile(path, caSink);
break;
}
},
}, method.raw);
return ContentAddress {
switch (method) {
case FileIngestionMethod::Recursive:
dumpPath(path, caSink);
break;
case FileIngestionMethod::Flat:
readFile(path, caSink);
break;
}
auto hash = caSink.finish().first;
return FixedOutputHash{
.method = method,
.hash = caSink.finish().first,
.hash = hash,
};
}
+8 -15
View File
@@ -5,7 +5,8 @@
#include "pathlocks.hh"
#include "store-api.hh"
#include "indirect-root-store.hh"
#include "local-fs-store.hh"
#include "gc-store.hh"
#include "sync.hh"
#include "util.hh"
@@ -67,9 +68,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
std::string doc() override;
};
class LocalStore : public virtual LocalStoreConfig
, public virtual IndirectRootStore
, public virtual GcStore
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
{
private:
@@ -166,7 +165,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryStaticPartialDerivationOutputMap(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
@@ -210,12 +209,6 @@ private:
public:
/**
* Implementation of IndirectRootStore::addIndirectRoot().
*
* The weak reference merely is a symlink to `path' from
* /nix/var/nix/gcroots/auto/<hash of `path'>.
*/
void addIndirectRoot(const Path & path) override;
private:
@@ -352,13 +345,13 @@ private:
void signRealisation(Realisation &);
// XXX: Make a generic `Store` method
ContentAddress hashCAPath(
const ContentAddressMethod & method,
FixedOutputHash hashCAPath(
const FileIngestionMethod & method,
const HashType & hashType,
const StorePath & path);
ContentAddress hashCAPath(
const ContentAddressMethod & method,
FixedOutputHash hashCAPath(
const FileIngestionMethod & method,
const HashType & hashType,
const Path & path,
const std::string_view pathHash
+4 -13
View File
@@ -52,8 +52,10 @@ std::map<StorePath, StorePath> makeContentAddressed(
dstStore,
path.name(),
FixedOutputInfo {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
},
.references = std::move(refs),
},
Hash::dummy,
@@ -78,15 +80,4 @@ std::map<StorePath, StorePath> makeContentAddressed(
return remappings;
}
StorePath makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePath & fromPath)
{
auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath });
auto i = remappings.find(fromPath);
assert(i != remappings.end());
return i->second;
}
}
+1 -12
View File
@@ -5,20 +5,9 @@
namespace nix {
/** Rewrite a closure of store paths to be completely content addressed.
*/
std::map<StorePath, StorePath> makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePathSet & rootPaths);
/** Rewrite a closure of a store path to be completely content addressed.
*
* This is a convenience function for the case where you only have one root path.
*/
StorePath makeContentAddressed(
Store & srcStore,
Store & dstStore,
const StorePath & rootPath);
const StorePathSet & storePaths);
}
+35 -26
View File
@@ -310,34 +310,43 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{
auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_);
auto outputsOpt = std::visit(overloaded {
[&](const OutputsSpec::All &) {
// Keep all outputs
return std::move(outputsOpt_);
},
[&](const OutputsSpec::Names & names) {
// Get just those mentioned by name
std::map<std::string, std::optional<StorePath>> outputsOpt;
for (auto & output : names) {
auto * pOutputPathOpt = get(outputsOpt_, output);
if (!pOutputPathOpt)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store.printStorePath(bfd.drvPath), output);
outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt));
}
return outputsOpt;
},
}, bfd.outputs.raw());
auto & evalStore = evalStore_ ? *evalStore_ : store;
OutputPathMap outputs;
for (auto & [outputName, outputPathOpt] : outputsOpt) {
if (!outputPathOpt)
throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName);
auto & outputPath = *outputPathOpt;
outputs.insert_or_assign(outputName, outputPath);
auto drv = evalStore.readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(store, drv);
auto drvOutputs = drv.outputsAndOptPaths(store);
auto outputNames = std::visit(overloaded {
[&](const OutputsSpec::All &) {
StringSet names;
for (auto & [outputName, _] : drv.outputs)
names.insert(outputName);
return names;
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, bfd.outputs.raw());
for (auto & output : outputNames) {
auto outputHash = get(outputHashes, output);
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store.printStorePath(bfd.drvPath), output);
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
DrvOutput outputId { *outputHash, output };
auto realisation = store.queryRealisation(outputId);
if (!realisation)
throw MissingRealisation(outputId);
outputs.insert_or_assign(output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign(output, *drvOutput->second);
}
}
return outputs;
}
+7 -14
View File
@@ -29,14 +29,14 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef
return std::nullopt;
return std::visit(overloaded {
[&](const TextIngestionMethod &) -> ContentAddressWithReferences {
[&](const TextHash & th) -> ContentAddressWithReferences {
assert(references.count(path) == 0);
return TextInfo {
.hash = ca->hash,
.hash = th,
.references = references,
};
},
[&](const FileIngestionMethod & m2) -> ContentAddressWithReferences {
[&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
@@ -44,15 +44,14 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef
refs.erase(path);
}
return FixedOutputInfo {
.method = m2,
.hash = ca->hash,
.hash = foh,
.references = {
.others = std::move(refs),
.self = hasSelfReference,
},
};
},
}, ca->method.raw);
}, ca->raw);
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
@@ -111,19 +110,13 @@ ValidPathInfo::ValidPathInfo(
std::visit(overloaded {
[this](TextInfo && ti) {
this->references = std::move(ti.references);
this->ca = ContentAddress {
.method = TextIngestionMethod {},
.hash = std::move(ti.hash),
};
this->ca = std::move((TextHash &&) ti);
},
[this](FixedOutputInfo && foi) {
this->references = std::move(foi.references.others);
if (foi.references.self)
this->references.insert(path);
this->ca = ContentAddress {
.method = std::move(foi.method),
.hash = std::move(foi.hash),
};
this->ca = std::move((FixedOutputHash &&) foi);
},
}, std::move(ca).raw);
}
+2 -7
View File
@@ -5,7 +5,6 @@
#include "hash.hh"
#include "path.hh"
#include "derived-path.hh"
#include <nlohmann/json_fwd.hpp>
#include "comparator.hh"
#include "crypto.hh"
@@ -144,13 +143,9 @@ class MissingRealisation : public Error
{
public:
MissingRealisation(DrvOutput & outputId)
: MissingRealisation(outputId.outputName, outputId.strHash())
{}
MissingRealisation(std::string_view drv, std::string outputName)
: Error( "cannot operate on output '%s' of the "
: Error( "cannot operate on an output of the "
"unbuilt derivation '%s'",
outputName,
drv)
outputId.to_string())
{}
};
-31
View File
@@ -1,6 +1,5 @@
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "pool.hh"
namespace nix {
@@ -95,34 +94,4 @@ struct RemoteStore::Connection
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
};
/**
* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
* the connection as bad (causing it to be closed) if a non-daemon
* exception is thrown before the handle is closed. Such an exception
* causes a deviation from the expected protocol and therefore a
* desynchronization between the client and daemon.
*/
struct RemoteStore::ConnectionHandle
{
Pool<RemoteStore::Connection>::Handle handle;
bool daemonException = false;
ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
: handle(std::move(handle))
{ }
ConnectionHandle(ConnectionHandle && h)
: handle(std::move(h.handle))
{ }
~ConnectionHandle();
RemoteStore::Connection & operator * () { return *handle; }
RemoteStore::Connection * operator -> () { return &*handle; }
void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
void withFramedSink(std::function<void(Sink & sink)> fun);
};
}
+59 -35
View File
@@ -159,25 +159,49 @@ void RemoteStore::setOptions(Connection & conn)
}
RemoteStore::ConnectionHandle::~ConnectionHandle()
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
the connection as bad (causing it to be closed) if a non-daemon
exception is thrown before the handle is closed. Such an exception
causes a deviation from the expected protocol and therefore a
desynchronization between the client and daemon. */
struct ConnectionHandle
{
if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
debug("closing daemon connection because of an exception");
Pool<RemoteStore::Connection>::Handle handle;
bool daemonException = false;
ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
: handle(std::move(handle))
{ }
ConnectionHandle(ConnectionHandle && h)
: handle(std::move(h.handle))
{ }
~ConnectionHandle()
{
if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
debug("closing daemon connection because of an exception");
}
}
}
void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush)
{
auto ex = handle->processStderr(sink, source, flush);
if (ex) {
daemonException = true;
std::rethrow_exception(ex);
RemoteStore::Connection * operator -> () { return &*handle; }
RemoteStore::Connection & operator * () { return *handle; }
void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
{
auto ex = handle->processStderr(sink, source, flush);
if (ex) {
daemonException = true;
std::rethrow_exception(ex);
}
}
}
void withFramedSink(std::function<void(Sink & sink)> fun);
};
RemoteStore::ConnectionHandle RemoteStore::getConnection()
ConnectionHandle RemoteStore::getConnection()
{
return ConnectionHandle(connections->get());
}
@@ -354,36 +378,27 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
}
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
{
if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
if (!evalStore_) {
auto conn(getConnection());
conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, *conn);
} else {
auto & evalStore = *evalStore_;
auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path);
// union with the first branch overriding the statically-known ones
// when non-`std::nullopt`.
for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) {
if (optPath)
outputs.insert_or_assign(std::move(outputName), std::move(optPath));
else
outputs.insert({std::move(outputName), std::nullopt});
}
return outputs;
}
auto conn(getConnection());
conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, *conn);
} else {
auto & evalStore = evalStore_ ? *evalStore_ : *this;
// Fallback for old daemon versions.
// For floating-CA derivations (and their co-dependencies) this is an
// under-approximation as it only returns the paths that can be inferred
// from the derivation itself (and not the ones that are known because
// the have been built), but as old stores don't handle floating-CA
// derivations this shouldn't matter
return evalStore.queryStaticPartialDerivationOutputMap(path);
auto derivation = readDerivation(path);
auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
std::map<std::string, std::optional<StorePath>> ret;
for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
ret.emplace(outputName, outputAndPath.second);
}
return ret;
}
}
@@ -822,6 +837,15 @@ void RemoteStore::addTempRoot(const StorePath & path)
}
void RemoteStore::addIndirectRoot(const Path & path)
{
auto conn(getConnection());
conn->to << WorkerProto::Op::AddIndirectRoot << path;
conn.processStderr();
readInt(conn->from);
}
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
@@ -1066,7 +1090,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
return nullptr;
}
void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
void ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
{
(*this)->to.flush();
+5 -3
View File
@@ -17,6 +17,7 @@ class Pid;
struct FdSink;
struct FdSource;
template<typename T> class Pool;
struct ConnectionHandle;
struct RemoteStoreConfig : virtual StoreConfig
{
@@ -62,7 +63,7 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@@ -126,6 +127,8 @@ public:
void addTempRoot(const StorePath & path) override;
void addIndirectRoot(const Path & path) override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
@@ -179,8 +182,6 @@ protected:
void setOptions() override;
struct ConnectionHandle;
ConnectionHandle getConnection();
friend struct ConnectionHandle;
@@ -198,4 +199,5 @@ private:
std::shared_ptr<Store> evalStore);
};
}
+4 -8
View File
@@ -1,6 +1,5 @@
#include "ssh-store-config.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "remote-store.hh"
#include "remote-store-connection.hh"
#include "remote-fs-accessor.hh"
@@ -62,7 +61,7 @@ public:
std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ unsupported("getBuildLogExact"); }
protected:
private:
struct Connection : RemoteStore::Connection
{
@@ -94,12 +93,9 @@ protected:
ref<RemoteStore::Connection> SSHStore::openConnection()
{
auto conn = make_ref<Connection>();
std::string command = remoteProgram + " --stdio";
if (remoteStore.get() != "")
command += " --store " + shellEscape(remoteStore.get());
conn->sshConn = master.startCommand(command);
conn->sshConn = master.startCommand(
fmt("%s --stdio", remoteProgram)
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
return conn;
+1 -4
View File
@@ -42,10 +42,7 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
}
bool SSHMaster::isMasterRunning() {
Strings args = {"-O", "check", host};
addCommonSSHOpts(args);
auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true});
auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
return res.first == 0;
}
+19 -43
View File
@@ -184,15 +184,15 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
{
if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name);
} else {
assert(info.references.size() == 0);
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ makeFileIngestionPrefix(info.method)
+ info.hash.to_string(Base16, true) + ":"),
+ makeFileIngestionPrefix(info.hash.method)
+ info.hash.hash.to_string(Base16, true) + ":"),
name);
}
}
@@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.type == htSHA256);
assert(info.hash.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = info.references,
.self = false,
}),
info.hash,
info.hash.hash,
name);
}
@@ -232,8 +232,10 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
FixedOutputInfo caInfo {
.method = method,
.hash = h,
.hash = {
.method = method,
.hash = h,
},
.references = {},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
@@ -246,8 +248,8 @@ StorePath Store::computeStorePathForText(
const StorePathSet & references) const
{
return makeTextPath(name, TextInfo {
.hash = hashString(htSHA256, s),
.references = references,
{ .hash = hashString(htSHA256, s) },
references,
});
}
@@ -439,8 +441,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
*this,
name,
FixedOutputInfo {
.method = method,
.hash = hash,
.hash = {
.method = method,
.hash = hash,
},
.references = {},
},
narHash,
@@ -492,50 +496,22 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
std::map<std::string, std::optional<StorePath>> Store::queryStaticPartialDerivationOutputMap(const StorePath & path)
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
{
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = readInvalidDerivation(path);
for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) {
for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) {
outputs.emplace(outputName, output.second);
}
return outputs;
}
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : *this;
auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path);
if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
return outputs;
auto drv = evalStore.readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto & [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation) {
outputs.insert_or_assign(outputName, realisation->outPath);
} else {
// queryStaticPartialDerivationOutputMap is not guaranteed
// to return std::nullopt for outputs which are not
// statically known.
outputs.insert({outputName, std::nullopt});
}
}
return outputs;
}
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
auto resp = queryPartialDerivationOutputMap(path);
OutputPathMap result;
for (auto & [outName, optOutPath] : resp) {
if (!optOutPath)
throw MissingRealisation(printStorePath(path), outName);
throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path));
result.insert_or_assign(outName, *optOutPath);
}
return result;
+5 -16
View File
@@ -99,8 +99,6 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public Config
{
typedef std::map<std::string, std::string> Params;
using Config::Config;
StoreConfig() = delete;
@@ -155,6 +153,10 @@ struct StoreConfig : public Config
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
{
public:
typedef std::map<std::string, std::string> Params;
protected:
struct PathInfoCacheValue {
@@ -423,20 +425,7 @@ public:
* derivation. All outputs are mentioned so ones mising the mapping
* are mapped to `std::nullopt`.
*/
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore = nullptr);
/**
* Like `queryPartialDerivationOutputMap` but only considers
* statically known output paths (i.e. those that can be gotten from
* the derivation itself.
*
* Just a helper function for implementing
* `queryPartialDerivationOutputMap`.
*/
virtual std::map<std::string, std::optional<StorePath>> queryStaticPartialDerivationOutputMap(
const StorePath & path);
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
/**
* Query the mapping outputName=>outputPath for the given derivation.
+3 -3
View File
@@ -81,7 +81,7 @@ TEST_JSON(DerivationTest, caFixedFlat,
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = {
.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
@@ -95,7 +95,7 @@ TEST_JSON(DerivationTest, caFixedNAR,
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = {
.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
@@ -109,7 +109,7 @@ TEST_JSON(DynDerivationTest, caFixedText,
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = {
.ca = TextHash {
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
}),
-10
View File
@@ -1,5 +1,4 @@
#include "uds-remote-store.hh"
#include "worker-protocol.hh"
#include <sys/types.h>
#include <sys/stat.h>
@@ -78,15 +77,6 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
}
void UDSRemoteStore::addIndirectRoot(const Path & path)
{
auto conn(getConnection());
conn->to << WorkerProto::Op::AddIndirectRoot << path;
conn.processStderr();
readInt(conn->from);
}
static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regUDSRemoteStore;
}
+3 -15
View File
@@ -3,13 +3,13 @@
#include "remote-store.hh"
#include "remote-store-connection.hh"
#include "indirect-root-store.hh"
#include "local-fs-store.hh"
namespace nix {
struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig
{
UDSRemoteStoreConfig(const Params & params)
UDSRemoteStoreConfig(const Store::Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(params)
, RemoteStoreConfig(params)
@@ -21,9 +21,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
std::string doc() override;
};
class UDSRemoteStore : public virtual UDSRemoteStoreConfig
, public virtual IndirectRootStore
, public virtual RemoteStore
class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore
{
public:
@@ -41,16 +39,6 @@ public:
void narFromPath(const StorePath & path, Sink & sink) override
{ LocalFSStore::narFromPath(path, sink); }
/**
* Implementation of `IndirectRootStore::addIndirectRoot()` which
* delegates to the remote store.
*
* The idea is that the client makes the direct symlink, so it is
* owned managed by the client's user account, and the server makes
* the indirect symlink.
*/
void addIndirectRoot(const Path & path) override;
private:
struct Connection : RemoteStore::Connection
+4 -2
View File
@@ -220,8 +220,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
std::string name = *i++;
cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo {
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
.hash = {
.method = method,
.hash = Hash::parseAny(hash, hashAlgo),
},
.references = {},
})));
}
+4 -2
View File
@@ -45,8 +45,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
*store,
std::move(*namePart),
FixedOutputInfo {
.method = std::move(ingestionMethod),
.hash = std::move(hash),
.hash = {
.method = std::move(ingestionMethod),
.hash = std::move(hash),
},
.references = {},
},
narHash,
+2 -4
View File
@@ -386,10 +386,8 @@ struct CmdFlakeCheck : FlakeCommand
auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) {
try {
state->forceValue(v, pos);
if (!v.isLambda()) {
throw Error("overlay is not a function, but %s instead", showType(v));
}
if (v.lambda.fun->hasFormals()
if (!v.isLambda()
|| v.lambda.fun->hasFormals()
|| !argHasName(v.lambda.fun->arg, "final"))
throw Error("overlay does not take an argument named 'final'");
auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
+2 -17
View File
@@ -71,6 +71,8 @@ inputs.nixpkgs = {
Here are some examples of flake references in their URL-like representation:
* `.`: The flake in the current directory.
* `/home/alice/src/patchelf`: A flake in some other directory.
* `nixpkgs`: The `nixpkgs` entry in the flake registry.
* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs`
entry in the flake registry, with its Git revision overridden to a
@@ -91,23 +93,6 @@ Here are some examples of flake references in their URL-like representation:
* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball
flake.
## Path-like syntax
Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity).
The semantic of such a path is as follows:
* If the directory is part of a Git repository, then the input will be treated as a `git+file:` URL, otherwise it will be treated as a `path:` url;
* If the directory doesn't contain a `flake.nix` file, then Nix will search for such a file upwards in the file system hierarchy until it finds any of:
1. The Git repository root, or
2. The filesystem root (/), or
3. A folder on a different mount point.
### Examples
* `.`: The flake to which the current directory belongs to.
* `/home/alice/src/patchelf`: A flake in some other directory.
## Flake reference attributes
The following generic flake reference attributes are supported:
+12 -29
View File
@@ -352,7 +352,7 @@ void mainWrapped(int argc, char * * argv)
return;
}
if (argc == 2 && std::string(argv[1]) == "__dump-language") {
if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
experimentalFeatureSettings.experimentalFeatures = {
Xp::Flakes,
Xp::FetchClosure,
@@ -360,34 +360,17 @@ void mainWrapped(int argc, char * * argv)
evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://"));
auto res = nlohmann::json::object();
res["builtins"] = ({
auto builtinsJson = nlohmann::json::object();
auto builtins = state.baseEnv.values[0]->attrs;
for (auto & builtin : *builtins) {
auto b = nlohmann::json::object();
if (!builtin.value->isPrimOp()) continue;
auto primOp = builtin.value->primOp;
if (!primOp->doc) continue;
b["arity"] = primOp->arity;
b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(primOp->doc));
b["experimental-feature"] = primOp->experimentalFeature;
builtinsJson[state.symbols[builtin.name]] = std::move(b);
}
std::move(builtinsJson);
});
res["constants"] = ({
auto constantsJson = nlohmann::json::object();
for (auto & [name, info] : state.constantInfos) {
auto c = nlohmann::json::object();
if (!info.doc) continue;
c["doc"] = trim(stripIndentation(info.doc));
c["type"] = showType(info.type, false);
c["impure-only"] = info.impureOnly;
constantsJson[name] = std::move(c);
}
std::move(constantsJson);
});
auto builtins = state.baseEnv.values[0]->attrs;
for (auto & builtin : *builtins) {
auto b = nlohmann::json::object();
if (!builtin.value->isPrimOp()) continue;
auto primOp = builtin.value->primOp;
if (!primOp->doc) continue;
b["arity"] = primOp->arity;
b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(primOp->doc));
res[state.symbols[builtin.name]] = std::move(b);
}
logger->cout("%s", res);
return;
}
+1 -1
View File
@@ -63,7 +63,7 @@ The following types of installable are supported by most commands:
- [Nix file](#nix-file), optionally qualified by an attribute path
- [Nix expression](#nix-expression), optionally qualified by an attribute path
For most commands, if no installable is specified, `.` is assumed.
For most commands, if no installable is specified, `.` as assumed.
That is, Nix will operate on the default flake output attribute of the flake in the current directory.
### Flake output attribute
+5 -3
View File
@@ -71,8 +71,10 @@ std::tuple<StorePath, Hash> prefetchFile(
if (expectedHash) {
hashType = expectedHash->type;
storePath = store->makeFixedOutputPath(*name, FixedOutputInfo {
.method = ingestionMethod,
.hash = *expectedHash,
.hash = {
.method = ingestionMethod,
.hash = *expectedHash,
},
.references = {},
});
if (store->isValidPath(*storePath))
@@ -125,7 +127,7 @@ std::tuple<StorePath, Hash> prefetchFile(
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
storePath = info.path;
assert(info.ca);
hash = info.ca->hash;
hash = info.ca->getHash();
}
return {storePath.value(), hash.value()};
+12 -34
View File
@@ -6,48 +6,26 @@ R""(
```console
# nix profile list
Index: 0
Flake attribute: legacyPackages.x86_64-linux.gdb
Original flake URL: flake:nixpkgs
Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca
Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1
Index: 1
Flake attribute: packages.x86_64-linux.default
Original flake URL: flake:blender-bin
Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender
Store paths: /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2
0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1
1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027
2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0
```
Note that you can unambiguously rebuild a package from a profile
through its locked flake URL and flake attribute, e.g.
```console
# nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default
```
will build the package with index 1 shown above.
# Description
This command shows what packages are currently installed in a
profile. For each installed package, it shows the following
information:
profile. The output consists of one line per package, with the
following fields:
* `Index`: An integer that can be used to unambiguously identify the
package in invocations of `nix profile remove` and `nix profile
upgrade`.
* An integer that can be used to unambiguously identify the package in
invocations of `nix profile remove` and `nix profile upgrade`.
* `Flake attribute`: The flake output attribute path that provides the
package (e.g. `packages.x86_64-linux.hello`).
* The original ("unlocked") flake reference and output attribute path
used at installation time.
* `Original flake URL`: The original ("unlocked") flake reference
specified by the user when the package was first installed via `nix
profile install`.
* The locked flake reference to which the unlocked flake reference was
resolved.
* `Locked flake URL`: The locked flake reference to which the original
flake reference was resolved.
* `Store paths`: The store path(s) of the package.
* The store path(s) of the package.
)""
+20 -28
View File
@@ -21,7 +21,7 @@ struct ProfileElementSource
{
FlakeRef originalRef;
// FIXME: record original attrpath.
FlakeRef lockedRef;
FlakeRef resolvedRef;
std::string attrPath;
ExtendedOutputsSpec outputs;
@@ -168,7 +168,7 @@ struct ProfileManifest
}
}
nlohmann::json toJSON(Store & store) const
std::string toJSON(Store & store) const
{
auto array = nlohmann::json::array();
for (auto & element : elements) {
@@ -181,7 +181,7 @@ struct ProfileManifest
obj["priority"] = element.priority;
if (element.source) {
obj["originalUrl"] = element.source->originalRef.to_string();
obj["url"] = element.source->lockedRef.to_string();
obj["url"] = element.source->resolvedRef.to_string();
obj["attrPath"] = element.source->attrPath;
obj["outputs"] = element.source->outputs;
}
@@ -190,7 +190,7 @@ struct ProfileManifest
nlohmann::json json;
json["version"] = 2;
json["elements"] = array;
return json;
return json.dump();
}
StorePath build(ref<Store> store)
@@ -210,7 +210,7 @@ struct ProfileManifest
buildProfile(tempDir, std::move(pkgs));
writeFile(tempDir + "/manifest.json", toJSON(*store).dump());
writeFile(tempDir + "/manifest.json", toJSON(*store));
/* Add the symlink tree to the store. */
StringSink sink;
@@ -222,8 +222,10 @@ struct ProfileManifest
*store,
"profile",
FixedOutputInfo {
.method = FileIngestionMethod::Recursive,
.hash = narHash,
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narHash,
},
.references = {
.others = std::move(references),
// profiles never refer to themselves
@@ -347,7 +349,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
if (auto * info2 = dynamic_cast<ExtraPathInfoFlake *>(&*info)) {
element.source = ProfileElementSource {
.originalRef = info2->flake.originalRef,
.lockedRef = info2->flake.lockedRef,
.resolvedRef = info2->flake.resolvedRef,
.attrPath = info2->value.attrPath,
.outputs = info2->value.extendedOutputsSpec,
};
@@ -586,14 +588,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
assert(infop);
auto & info = *infop;
if (element.source->lockedRef == info.flake.lockedRef) continue;
if (element.source->resolvedRef == info.flake.resolvedRef) continue;
printInfo("upgrading '%s' from flake '%s' to '%s'",
element.source->attrPath, element.source->lockedRef, info.flake.lockedRef);
element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef);
element.source = ProfileElementSource {
.originalRef = installable->flakeRef,
.lockedRef = info.flake.lockedRef,
.resolvedRef = info.flake.resolvedRef,
.attrPath = info.value.attrPath,
.outputs = installable->extendedOutputsSpec,
};
@@ -633,7 +635,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
}
};
struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON
struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
{
std::string description() override
{
@@ -651,22 +653,12 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
{
ProfileManifest manifest(*getEvalState(), *profile);
if (json) {
std::cout << manifest.toJSON(*store).dump() << "\n";
} else {
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
if (i) logger->cout("");
logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s",
i,
element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL);
if (element.source) {
logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string());
logger->cout("Original flake URL: %s", element.source->originalRef.to_string());
logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string());
}
logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
}
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
logger->cout("%d %s %s %s", i,
element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-",
concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
}
}
};
+1 -1
View File
@@ -146,7 +146,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
auto req = FileTransferRequest(storePathsUrl);
auto res = getFileTransfer()->download(req);
auto state = std::make_unique<EvalState>(SearchPath{}, store);
auto state = std::make_unique<EvalState>(Strings(), store);
auto v = state->allocValue();
state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v);
Bindings & bindings(*state->allocBindings(0));
-1
View File
@@ -1,7 +1,6 @@
requireSandboxSupport
[[ $busybox =~ busybox ]] || skipTest "no busybox"
# Avoid store dir being inside sandbox build-dir
unset NIX_STORE_DIR
unset NIX_STATE_DIR
-27
View File
@@ -1,27 +0,0 @@
ca-tests := \
$(d)/build-with-garbage-path.sh \
$(d)/build.sh \
$(d)/concurrent-builds.sh \
$(d)/derivation-json.sh \
$(d)/duplicate-realisation-in-closure.sh \
$(d)/gc.sh \
$(d)/import-derivation.sh \
$(d)/new-build-cmd.sh \
$(d)/nix-copy.sh \
$(d)/nix-run.sh \
$(d)/nix-shell.sh \
$(d)/post-hook.sh \
$(d)/recursive.sh \
$(d)/repl.sh \
$(d)/selfref-gc.sh \
$(d)/signatures.sh \
$(d)/substitute.sh \
$(d)/why-depends.sh
install-tests-groups += ca
clean-files += \
$(d)/config.nix
test-deps += \
tests/ca/config.nix
+1 -1
View File
@@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then
COMMON_SH_SOURCED=1
source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh"
source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh"
if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
startDaemon
fi
+1 -1
View File
@@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then
COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1
export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) '
export PS4='+(${BASH_SOURCE[0]}:$LINENO) '
export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
export NIX_STORE_DIR
-3
View File
@@ -15,9 +15,6 @@ if test -n "$dot"; then
$dot < $TEST_ROOT/graph
fi
# Test GraphML graph generation
nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml
outPath=$(nix-store -rvv "$drvPath") || fail "build failed"
# Test Graphviz graph generation.
-11
View File
@@ -1,11 +0,0 @@
dyn-drv-tests := \
$(d)/text-hashed-output.sh \
$(d)/recursive-mod-json.sh
install-tests-groups += dyn-drv
clean-files += \
$(d)/config.nix
test-deps += \
tests/dyn-drv/config.nix
+2 -73
View File
@@ -33,43 +33,20 @@ clearStore
[ ! -e $nonCaPath ]
[ -e $caPath ]
clearStore
# The daemon will reject input addressed paths unless configured to trust the
# cache key or the user. This behavior should be covered by another test, so we
# skip this part when using the daemon.
if [[ "$NIX_REMOTE" != "daemon" ]]; then
# If we want to return a non-CA path, we have to be explicit about it.
expectStderr 1 nix eval --raw --no-require-sigs --expr "
# In impure mode, we can use non-CA paths.
[[ $(nix eval --raw --no-require-sigs --impure --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
}
" | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false."
# TODO: Should the closure be rejected, despite single user mode?
# [ ! -e $nonCaPath ]
[ ! -e $caPath ]
# We can use non-CA paths when we ask explicitly.
[[ $(nix eval --raw --no-require-sigs --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
inputAddressed = true;
}
") = $nonCaPath ]]
[ -e $nonCaPath ]
[ ! -e $caPath ]
fi
[ ! -e $caPath ]
# 'toPath' set to empty string should fail but print the expected path.
expectStderr 1 nix eval -v --json --expr "
builtins.fetchClosure {
@@ -82,10 +59,6 @@ expectStderr 1 nix eval -v --json --expr "
# If fromPath is CA, then toPath isn't needed.
nix copy --to file://$cacheDir $caPath
clearStore
[ ! -e $caPath ]
[[ $(nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
@@ -93,8 +66,6 @@ clearStore
}
") = $caPath ]]
[ -e $caPath ]
# Check that URL query parameters aren't allowed.
clearStore
narCache=$TEST_ROOT/nar-cache
@@ -106,45 +77,3 @@ rm -rf $narCache
}
")
(! [ -e $narCache ])
# If toPath is specified but wrong, we check it (only) when the path is missing.
clearStore
badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!')
[ ! -e $badPath ]
expectStderr 1 nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $nonCaPath;
toPath = $badPath;
}
" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected"
[ ! -e $badPath ]
# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it.
# It would be nice for this to fail, but checking it would be too(?) slow.
[ -e $caPath ]
[[ $(nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $badPath;
toPath = $caPath;
}
") = $caPath ]]
# However, if the output address is unexpected, we can report it
expectStderr 1 nix eval -v --raw --expr "
builtins.fetchClosure {
fromStore = \"file://$cacheDir\";
fromPath = $caPath;
inputAddressed = true;
}
" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*'
Loaded 100 of 186 files, more files were not shown because too many files have changed in this diff. Show more