Compare commits

..
Author SHA1 Message Date
Eelco Dolstra 3f53a435cb Factor out the local NAR cache into its own class 2026-01-14 22:22:44 +01:00
318 changed files with 1857 additions and 4230 deletions

No files matched your search

@@ -24,8 +24,8 @@ inputs:
description: "Github token"
required: true
use_cache:
description: "Whether to setup github actions cache (not implemented currently)"
default: false
description: "Whether to setup magic-nix-cache"
default: true
required: false
runs:
using: "composite"
@@ -122,3 +122,10 @@ runs:
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
extra-conf: ${{ inputs.extra_nix_config }}
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
if: ${{ inputs.use_cache == 'true' }}
with:
diagnostic-endpoint: ''
use-flakehub: false
use-gha-cache: true
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
-2
View File
@@ -1,7 +1,5 @@
# Default meson build dir
/build
# Meson creates this file too
src/.wraplock
# /tests/functional/
/tests/functional/common/subst-vars.sh
@@ -1,8 +0,0 @@
---
synopsis: "C API: New store API methods"
prs: [14766]
---
The C API now includes additional methods:
- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
@@ -1,10 +0,0 @@
---
synopsis: "New setting `ignore-gc-delete-failure` for local stores"
prs: [15054]
---
A new local store setting [`ignore-gc-delete-failure`](@docroot@/store/types/local-store.md#store-local-store-ignore-gc-delete-failure) has been added.
When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths.
This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store.
This setting is experimental and requires the [`local-overlay-store`](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store) experimental feature.
-15
View File
@@ -1,15 +0,0 @@
---
synopsis: Support HTTPS binary caches using mTLS (client certificate) authentication
issues: [13002]
prs: [13030]
---
Added support for `tls-certificate` and `tls-private-key` options in substituter URLs.
Example:
```
https://substituter.invalid?tls-certificate=/path/to/cert.pem&tls-private-key=/path/to/key.pem
```
When these options are configured, Nix will use this certificate/private key pair to authenticate to the server.
-2
View File
@@ -135,9 +135,7 @@
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
- [Store Path Specification](protocols/store-path.md)
- [Nix Archive (NAR) Format](protocols/nix-archive/index.md)
- [Nix Cache Info Format](protocols/nix-cache-info.md)
- [Derivation "ATerm" file format](protocols/derivation-aterm.md)
- [Nix32 Encoding](protocols/nix32.md)
- [C API](c-api.md)
- [Glossary](glossary.md)
- [Development](development/index.md)
@@ -57,6 +57,11 @@ Most Nix commands interpret the following environment variables:
Overrides the location of the Nix store (default `prefix/store`).
- <span id="env-NIX_DATA_DIR">[`NIX_DATA_DIR`](#env-NIX_DATA_DIR)</span>
Overrides the location of the Nix static data directory (default
`prefix/share`).
- <span id="env-NIX_LOG_DIR">[`NIX_LOG_DIR`](#env-NIX_LOG_DIR)</span>
Overrides the location of the Nix log directory (default
@@ -6,23 +6,14 @@ It is broken up into multiple Meson packages, which are optionally combined in a
There are no mandatory extra steps to the building process:
generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work.
```bash
git clone https://github.com/NixOS/nix.git
cd nix
meson setup build
cd build
ninja
(sudo) ninja install
```
The installation path can be specified by passing `-Dprefix=prefix`
to `meson setup build`. The default installation directory is `/usr/local`. You
The installation path can be specified by passing the `-Dprefix=prefix`
to `configure`. The default installation directory is `/usr/local`. You
can change this to any location you like. You must have write permission
to the *prefix* path.
Nix keeps its *store* (the place where packages are stored) in
`/nix/store` by default. This can be changed using
`-Dlibstore:store-dir=path`.
`-Dstore-dir=path`.
> **Warning**
>
@@ -338,7 +338,7 @@ Here is more information on the `output*` attributes, and what values they may b
This will specify the output hash of the single output of a [fixed-output derivation].
The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/).
The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding.
The "nix32" encoding is an adaptation of base-32 encoding.
> **Note**
>
@@ -19,16 +19,17 @@ whatever port you like:
$ nix-serve -p 8080
```
To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client:
To check whether it works, try the following on the client:
```console
$ curl http://avalon:8080/nix-cache-info
StoreDir: /nix/store
WantMassQuery: 1
Priority: 30
```
When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist.
which should print something like:
StoreDir: /nix/store
WantMassQuery: 1
Priority: 30
On the client side, you can tell Nix to use your binary cache using
`--substituters`, e.g.:
@@ -24,7 +24,7 @@ description: |
The format follows this pattern: `${digest}-${name}`
- **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
- **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
- **name**: The package name and optional version/suffix information
type: string
@@ -1,55 +0,0 @@
# Nix Cache Info Format
The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`).
MIME type: `text/x-nix-cache-info`
## Format
Line-based key-value format:
```
Key: value
```
Leading and trailing whitespace is trimmed from values.
Lines without a colon are ignored.
Unknown keys are silently ignored.
## Fields
### `StoreDir`
The Nix store directory path that this cache was built for (e.g., `/nix/store`).
If present, Nix verifies that this matches the client's store directory:
```
error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/store', not '/home/user/nix/store'
```
### `WantMassQuery`
`1` or `0`. Sets the default for [`want-mass-query`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-want-mass-query).
### `Priority`
Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority).
## Example
```
StoreDir: /nix/store
WantMassQuery: 1
Priority: 30
```
## Caching Behavior
Nix caches `nix-cache-info` in the [cache directory](@docroot@/command-ref/env-common.md#env-NIX_CACHE_HOME) with a 7-day TTL.
## See Also
- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md)
- [Serving a Nix store via HTTP](@docroot@/package-management/binary-cache-substituter.md)
- [`substituters`](@docroot@/command-ref/conf-file.md#conf-substituters)
-19
View File
@@ -1,19 +0,0 @@
# Nix32 Encoding
Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute.
## Alphabet
The Nix32 alphabet consists of these 32 characters:
```
0 1 2 3 4 5 6 7 8 9 a b c d f g h i j k l m n p q r s v w x y z
```
The letters `e`, `o`, `u`, and `t` are omitted.
## Byte Order
Nix32 encoding processes the hash bytes from the end (last byte first), while base-16 encoding processes from the beginning (first byte first).
Consequently, the string sort order is determined primarily by the first bytes for base-16, and by the last bytes for Nix32.
+5 -4
View File
@@ -20,11 +20,12 @@ where
- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory)
- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`.
- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`
Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md).
For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf).
For the definition of the hash compression algorithm, please refer to the section 5.1 of
the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf), which also defines the
specifics of base-32 encoding. Note that base-32 encoding processes the hash bytestring from
the end, while base-16 processes in from the beginning.
## Fingerprint
+1 -1
View File
@@ -31,7 +31,7 @@ A store path is rendered to a file system path as the concatenation of
- [Store directory](#store-directory) (typically `/nix/store`)
- Path separator (`/`)
- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
- Hyphen (`-`)
- Name
Generated
+5 -5
View File
@@ -63,15 +63,15 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1769089682,
"narHash": "sha256-Xu+7iYcAuOvsI2wdkUcIEmkqEJbvvE6n7qR9QNjJyP4=",
"rev": "078d69f03934859a181e81ba987c2bb033eebfc5",
"lastModified": 1763948260,
"narHash": "sha256-zZk7fn2ARAqmLwaYTpxBJmj81KIdz11NiWt7ydHHD/M=",
"rev": "1c8ba8d3f7634acac4a2094eef7c32ad9106532c",
"type": "tarball",
"url": "https://releases.nixos.org/nixos/25.11/nixos-25.11.4506.078d69f03934/nixexprs.tar.xz"
"url": "https://releases.nixos.org/nixos/25.05/nixos-25.05.813095.1c8ba8d3f763/nixexprs.tar.xz"
},
"original": {
"type": "tarball",
"url": "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz"
"url": "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz"
}
},
"nixpkgs-23-11": {
+1 -1
View File
@@ -1,7 +1,7 @@
{
description = "The purely functional package manager";
inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.11/nixexprs.tar.xz";
inputs.nixpkgs.url = "https://channels.nixos.org/nixos-25.05/nixexprs.tar.xz";
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
+2 -9
View File
@@ -88,23 +88,16 @@
''^tests/functional/lang/eval-fail-path-slash\.nix$''
''^tests/functional/lang/eval-fail-toJSON-non-utf-8\.nix$''
''^tests/functional/lang/eval-fail-set\.nix$''
# Language tests, don't churn the formatting of strings
''^tests/functional/lang/eval-fail-fromTOML-overflow\.nix$''
''^tests/functional/lang/eval-fail-fromTOML-underflow\.nix$''
''^tests/functional/lang/eval-fail-bad-string-interpolation-3\.nix$''
''^tests/functional/lang/eval-fail-bad-string-interpolation-4\.nix$''
''^tests/functional/lang/eval-okay-regex-match2\.nix$''
];
};
clang-format = {
enable = true;
# https://github.com/cachix/git-hooks.nix/pull/532
package = pkgs.llvmPackages_21.clang-tools;
package = pkgs.llvmPackages_latest.clang-tools;
excludes = [
# We don't want to format test data
# ''tests/(?!nixos/).*\.nix''
"^src/[^/]*-tests/data/.*$"
''^src/[^/]*-tests/data/.*$''
# Don't format vendored code
''^doc/manual/redirects\.js$''
+1 -1
View File
@@ -11,7 +11,7 @@ ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
KillMode=process
LimitNOFILE=1048576
TasksMax=1048576
Delegate=
Delegate=yes
[Install]
WantedBy=multi-user.target
@@ -1,7 +1,7 @@
# Clang gets grumpy about missing libasan symbols if -shared-libasan is not
# passed when building shared libs, at least on Linux
if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option(
'b_sanitize',
'b_sanitize',
))
add_project_link_arguments('-shared-libasan', language : 'cpp')
endif
+2 -4
View File
@@ -155,14 +155,12 @@ let
];
};
mesonBuildLayer = finalAttrs: prevAttrs: rec {
mesonBuildLayer = finalAttrs: prevAttrs: {
nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [
pkg-config
];
separateDebugInfo = !stdenv.hostPlatform.isStatic;
# needed by separateDebugInfo
# SEE: https://github.com/NixOS/nixpkgs/pull/394674/commits/a4d355342976e9e9823fb94f133bc43ebec9da5b
__structuredAttrs = separateDebugInfo;
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
};
mesonLibraryLayer = finalAttrs: prevAttrs: {
+26 -6
View File
@@ -30,13 +30,33 @@ scope: {
NIX_CFLAGS_COMPILE = "-DINITIAL_MARK_STACK_SIZE=1048576";
});
curl = pkgs.curl.override {
http3Support = !pkgs.stdenv.hostPlatform.isWindows;
};
lowdown = pkgs.lowdown.overrideAttrs (prevAttrs: rec {
version = "2.0.2";
src = pkgs.fetchurl {
url = "https://kristaps.bsd.lv/lowdown/snapshots/lowdown-${version}.tar.gz";
hash = "sha512-cfzhuF4EnGmLJf5EGSIbWqJItY3npbRSALm+GarZ7SMU7Hr1xw0gtBFMpOdi5PBar4TgtvbnG4oRPh+COINGlA==";
};
nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ];
postInstall =
lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
(prevAttrs.postInstall or "");
patches = [ ];
});
libblake3 = pkgs.libblake3.override {
useTBB = !(stdenv.hostPlatform.isWindows || stdenv.hostPlatform.isStatic);
};
# TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
toml11 =
if lib.versionAtLeast pkgs.toml11.version "4.4.0" then
pkgs.toml11
else
pkgs.toml11.overrideAttrs rec {
version = "4.4.0";
src = pkgs.fetchFromGitHub {
owner = "ToruNiina";
repo = "toml11";
tag = "v${version}";
hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
};
};
# TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
boost =
+4 -4
View File
@@ -131,7 +131,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
availableComponents = lib.filterAttrs (
k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v
k: v: lib.meta.availableOn pkgs.hostPlatform v
) allComponents;
activeComponents = buildInputsClosureCond isInternal (
@@ -300,7 +300,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
lib.filter (x: !isInternal x) (
lib.lists.concatMap (
# Nix manual has a build-time dependency on nix, but we
# don't want to do a native build just to enter the cross
# don't want to do a native build just to enter the ross
# dev shell.
#
# TODO: think of a more principled fix for this.
@@ -323,7 +323,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
pkgs.buildPackages.shellcheck
pkgs.buildPackages.include-what-you-use
]
++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
lib.hiPrio pkgs.buildPackages.clang-tools
)
@@ -341,7 +341,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
buildInputs =
# TODO change Nixpkgs to mark gbenchmark as building on Windows
lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
-1
View File
@@ -38,7 +38,6 @@ escape_systemd_env() {
create_systemd_proxy_env() {
vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY"
for v in $vars; do
# shellcheck disable=SC2268
if [ "x${!v:-}" != "x" ]; then
echo "Environment=${v}=$(escape_systemd_env "${!v}")"
fi
-2
View File
@@ -300,8 +300,6 @@ MixDefaultProfile::MixDefaultProfile()
profile = getDefaultProfile().string();
}
static constexpr auto environmentVariablesCategory = "Options that change environment variables";
MixEnvironment::MixEnvironment()
: ignoreEnvironment(false)
{
+1 -1
View File
@@ -795,7 +795,7 @@ void NixRepl::loadFiles()
loadedFiles.clear();
for (auto & i : old) {
notice("Loading %1%...", PathFmt(i));
notice("Loading '%1%'...", i);
loadFile(i);
}
+2 -2
View File
@@ -43,7 +43,7 @@ RC_GTEST_FIXTURE_PROP(
auto * v = state.allocValue();
state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
auto d = state.coerceToSingleDerivedPath(noPos, *v, "", mockXpSettings, true);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "", mockXpSettings);
RC_ASSERT(SingleDerivedPath{b} == d);
}
@@ -57,7 +57,7 @@ RC_GTEST_FIXTURE_PROP(
auto * v = state.allocValue();
state.mkOutputString(*v, b, outPath, mockXpSettings);
auto d = state.coerceToSingleDerivedPath(noPos, *v, "", mockXpSettings, true);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "", mockXpSettings);
RC_ASSERT(SingleDerivedPath{b} == d);
}
+1 -1
View File
@@ -75,7 +75,7 @@ struct AttrDb
auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite");
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
+1 -1
View File
@@ -139,7 +139,7 @@ public:
, profileFd([&]() {
AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
if (!fd)
throw SysError("opening file %s", PathFmt(profileFile));
throw SysError("opening file %s", profileFile);
return fd;
}())
, posCache(state)
+55 -139
View File
@@ -2406,8 +2406,7 @@ BackedStringView EvalState::coerceToString(
std::string_view errorCtx,
bool coerceMore,
bool copyToStore,
bool canonicalizePath,
bool strict)
bool canonicalizePath)
{
auto _level = addCallDepth(pos);
@@ -2434,28 +2433,14 @@ BackedStringView EvalState::coerceToString(
auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
if (maybeString)
return std::move(*maybeString);
auto outPathAttr = v.attrs()->get(s.outPath);
if (!outPathAttr) {
auto i = v.attrs()->get(s.outPath);
if (!i) {
error<TypeError>(
"cannot coerce %1% to a string: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
try {
auto derivedPath = coerceToSingleDerivedPath(pos, v, errorCtx);
auto result = mkSingleDerivedPathStringRaw(derivedPath);
context.insert(
std::visit([](auto && o) -> NixStringContextElem { return std::move(o); }, std::move(derivedPath)));
return result;
} catch (Error & e) {
if (strict)
throw;
auto info = e.info();
info.msg = HintFmt("in a future version of Nix this will be an error: %s", Uncolored(info.msg.str()));
logWarning(info);
return coerceToString(
pos, *outPathAttr->value, context, errorCtx, coerceMore, copyToStore, canonicalizePath, strict);
}
return coerceToString(pos, *i->value, context, errorCtx, coerceMore, copyToStore, canonicalizePath);
}
if (v.type() == nExternal) {
@@ -2493,8 +2478,7 @@ BackedStringView EvalState::coerceToString(
"while evaluating one element of the list",
coerceMore,
copyToStore,
canonicalizePath,
strict);
canonicalizePath);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
@@ -2581,131 +2565,63 @@ EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & con
error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
}
SingleDerivedPath EvalState::coerceToSingleDerivedPath(
const PosIdx pos,
Value & v,
std::string_view errorCtx,
const ExperimentalFeatureSettings & xpSettings,
bool skipStringValidation)
std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(
const PosIdx pos, Value & v, std::string_view errorCtx, const ExperimentalFeatureSettings & xpSettings)
{
forceValue(v, pos);
NixStringContext context;
auto s = forceString(v, context, pos, errorCtx, xpSettings);
auto csize = context.size();
if (csize != 1)
error<EvalError>("string '%s' has %d entries in its context. It should only have exactly one entry", s, csize)
.withTrace(pos, errorCtx)
.debugThrow();
auto derivedPath = std::visit(
overloaded{
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { return std::move(o); },
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error<EvalError>(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s)
.withTrace(pos, errorCtx)
.debugThrow();
},
[&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); },
},
((NixStringContextElem &&) *context.begin()).raw);
return {
std::move(derivedPath),
std::move(s),
};
}
switch (v.type()) {
case nAttrs: {
if (!isDerivation(v)) {
error<TypeError>(
"cannot coerce a set that is not a package to a single derived path: %1%",
ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
auto outPathAttr = v.attrs()->get(s.outPath);
if (!outPathAttr) {
error<TypeError>(
"cannot coerce %1% to a single derived path: no 'outPath' attribute: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
}
auto outPathDerivedPath =
coerceToSingleDerivedPath(pos, *outPathAttr->value, errorCtx, xpSettings, skipStringValidation);
auto drvPathAttr = v.attrs()->get(s.drvPath);
if (drvPathAttr) {
auto drvPathDerivedPath =
coerceToSingleDerivedPath(pos, *drvPathAttr->value, errorCtx, xpSettings, skipStringValidation);
std::visit(
overloaded{
[&](const SingleDerivedPath::Built & built) {
if (*built.drvPath != drvPathDerivedPath) {
error<EvalError>(
"relative outPath '%s' does not refer to drvPath '%s'",
outPathDerivedPath.to_string(*store),
drvPathDerivedPath.to_string(*store))
.withTrace(pos, errorCtx)
.debugThrow();
}
},
[&](const SingleDerivedPath::Opaque &) {
error<EvalError>(
"relative outPath '%s' does not refer to drvPath '%s'",
outPathDerivedPath.to_string(*store),
drvPathDerivedPath.to_string(*store))
.withTrace(pos, errorCtx)
.debugThrow();
},
},
outPathDerivedPath.raw());
}
return outPathDerivedPath;
}
case nString: {
auto s = v.string_view();
NixStringContext context;
copyContext(v, context, xpSettings);
auto csize = context.size();
if (csize != 1)
error<EvalError>(
"string '%s' has %d entries in its context. It should only have exactly one entry", s, csize)
.withTrace(pos, errorCtx)
.debugThrow();
auto derivedPath = std::visit(
SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
{
auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_;
auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
if (s != sExpected) {
/* `std::visit` is used here just to provide a more precise
error message. */
std::visit(
overloaded{
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { return std::move(o); },
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error<EvalError>(
"string '%s' has a context which refers to a complete source and binary closure. This is not "
"supported at this time",
s)
[&](const SingleDerivedPath::Opaque & o) {
error<EvalError>("path string '%s' has context with the different path '%s'", s, sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
},
[&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); },
},
((NixStringContextElem &&) *context.begin()).raw);
if (!skipStringValidation) {
auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
if (s != sExpected) {
/* `std::visit` is used here just to provide a more precise
error message. */
std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & o) {
error<EvalError>("path string '%s' has context with the different path '%s'", s, sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
},
[&](const SingleDerivedPath::Built & b) {
error<EvalError>(
"string '%s' has context with the output '%s' from derivation '%s', but the string is "
"not the right placeholder for this derivation output. It should be '%s'",
s,
b.output,
b.drvPath->to_string(*store),
sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
}},
derivedPath.raw());
}
}
return derivedPath;
}
default:
error<TypeError>(
"cannot coerce %1% to a single derived path: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions))
.withTrace(pos, errorCtx)
.debugThrow();
[&](const SingleDerivedPath::Built & b) {
error<EvalError>(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s,
b.output,
b.drvPath->to_string(*store),
sExpected)
.withTrace(pos, errorCtx)
.debugThrow();
}},
derivedPath.raw());
}
return derivedPath;
}
// NOTE: This implementation must match eqValues!
+25 -99
View File
@@ -1,6 +1,5 @@
#include "nix/expr/get-drvs.hh"
#include "nix/expr/eval-inline.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/derivations.hh"
#include "nix/store/store-api.hh"
#include "nix/store/path-with-outputs.hh"
@@ -22,13 +21,13 @@ PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string
, attrs(nullptr)
, attrPath("")
{
auto [drvStorePath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
this->drvPath = {{Path{SingleDerivedPath::Opaque{drvStorePath}, drvStorePath}}};
this->drvPath = drvPath;
auto drv = store->derivationFromPath(drvStorePath);
auto drv = store->derivationFromPath(drvPath);
name = drvStorePath.name();
name = drvPath.name();
if (selectedOutputs.size() > 1)
throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
@@ -37,18 +36,10 @@ PackageInfo::PackageInfo(EvalState & state, ref<Store> store, const std::string
auto i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvStorePath), outputName);
auto & [outputName_, output] = *i;
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
auto & [outputName, output] = *i;
auto outStorePath = output.path(*store, drv.name, outputName_);
if (outStorePath) {
outPath = Path{SingleDerivedPath::Opaque{*outStorePath}, *outStorePath};
} else {
// CA derivation with unknown output path
outPath = Path{
SingleDerivedPath::Built{makeConstantStorePathRef(drvStorePath), outputName_},
std::nullopt};
}
outPath = {output.path(*store, drv.name, outputName)};
}
std::string PackageInfo::queryName() const
@@ -73,60 +64,26 @@ std::string PackageInfo::querySystem() const
return system;
}
std::optional<PackageInfo::Path> PackageInfo::queryDrvPathFlexible() const
std::optional<StorePath> PackageInfo::queryDrvPath() const
{
if (!drvPath && attrs) {
auto i = attrs->get(state->s.drvPath);
if (i) {
Value v;
v.mkAttrs(const_cast<Bindings *>(attrs));
std::optional<SingleDerivedPath> outPathDerivedPath;
if (auto i = attrs->get(state->s.drvPath)) {
NixStringContext context;
auto found = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
try {
// Validate derivation structure
outPathDerivedPath = state->coerceToSingleDerivedPath(
noPos, v, "while evaluating the derivation");
found.requireDerivation();
} catch (Error & e) {
auto info = e.info();
info.msg = HintFmt("in a future version of Nix this will be an error: %s", Uncolored(info.msg.str()));
logWarning(info);
}
if (outPathDerivedPath) {
// Validation passed. Now get drvPath.
auto drvPathDerivedPath = state->coerceToSingleDerivedPath(
i->pos, *i->value, "while evaluating the 'drvPath' attribute of a derivation");
if (auto * opaque = std::get_if<SingleDerivedPath::Opaque>(&drvPathDerivedPath.raw())) {
opaque->path.requireDerivation();
drvPath = {{Path{std::move(drvPathDerivedPath), opaque->path}}};
} else {
// Dynamic derivation - drvPath is itself a derivation output
drvPath = {{Path{std::move(drvPathDerivedPath), std::nullopt}}};
}
} else {
// Fall back to old behavior
NixStringContext context;
auto found = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
try {
found.requireDerivation();
} catch (Error & e) {
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
}
drvPath = {{Path{SingleDerivedPath::Opaque{found}, std::move(found)}}};
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
}
drvPath = {std::move(found)};
} else
drvPath = {std::nullopt};
}
return drvPath.value_or(std::nullopt);
}
std::optional<StorePath> PackageInfo::queryDrvPath() const
{
if (auto path = queryDrvPathFlexible())
return path->storePath;
return std::nullopt;
}
StorePath PackageInfo::requireDrvPath() const
{
if (auto drvPath = queryDrvPath())
@@ -134,49 +91,18 @@ StorePath PackageInfo::requireDrvPath() const
throw Error("derivation does not contain a 'drvPath' attribute");
}
PackageInfo::Path PackageInfo::queryOutPathFlexible() const
{
if (!outPath && attrs) {
Value v;
v.mkAttrs(const_cast<Bindings *>(attrs));
std::optional<SingleDerivedPath> derivedPath;
try {
derivedPath = state->coerceToSingleDerivedPath(
noPos, v, "while evaluating the derivation");
} catch (Error & e) {
auto info = e.info();
info.msg = HintFmt("in a future version of Nix this will be an error: %s", Uncolored(info.msg.str()));
logWarning(info);
}
if (derivedPath) {
if (auto * opaque = std::get_if<SingleDerivedPath::Opaque>(&derivedPath->raw())) {
outPath = Path{std::move(*derivedPath), opaque->path};
} else {
// Built path - no concrete output path available (placeholder)
outPath = Path{std::move(*derivedPath), std::nullopt};
}
} else {
// Fall back to old behavior
auto i = attrs->get(state->s.outPath);
NixStringContext context;
if (i) {
auto path = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the output path of a derivation");
outPath = Path{SingleDerivedPath::Opaque{path}, std::move(path)};
}
}
}
if (!outPath)
throw Error("derivation does not have an 'outPath' attribute");
return *outPath;
}
StorePath PackageInfo::queryOutPath() const
{
auto path = queryOutPathFlexible();
if (!path.storePath)
if (!outPath && attrs) {
auto i = attrs->get(state->s.outPath);
NixStringContext context;
if (i)
outPath = state->coerceToStorePath(
i->pos, *i->value, context, "while evaluating the output path of a derivation");
}
if (!outPath)
throw UnimplementedError("CA derivations are not yet supported");
return *path.storePath;
return *outPath;
}
PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
+21 -34
View File
@@ -715,9 +715,6 @@ public:
* string. If `coerceMore` is set, also converts nulls, integers,
* booleans and lists to a string. If `copyToStore` is set,
* referenced paths are copied to the Nix store as a side effect.
* If `strict` is set, ensures that when coercing an attrset with
* both `outPath` and `drvPath`, the string context for `outPath`
* agrees with the `drvPath` attribute.
*/
BackedStringView coerceToString(
const PosIdx pos,
@@ -726,8 +723,7 @@ public:
std::string_view errorCtx,
bool coerceMore = false,
bool copyToStore = true,
bool canonicalizePath = true,
bool strict = false);
bool canonicalizePath = true);
StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
@@ -746,33 +742,30 @@ public:
StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
/**
* Coerce to `SingleDerivedPath`.
*
* For strings: must be either a literal store path or a
* placeholder (see `DownstreamPlaceholder`). The string context
* must be exactly one element, which is either a
* `NixStringContextElem::Opaque` or `NixStringContextElem::Built`.
* (`NixStringContextElem::DrvDeep` is not permitted). The string
* is parsed based on the context --- the context is the source of
* truth, and ultimately tells us what we want, and then we ensure
* the string corresponds to it.
*
* For attrsets: must have an `outPath` attribute which is
* recursively coerced. If a `drvPath` attribute is present, it is
* also coerced and validated to agree with the `outPath` context.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*
* @param skipStringValidation Skip validating that the string
* matches the expected placeholder for the context. This is
* exposed for unit testing only to avoid store IO.
* Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only.
*/
SingleDerivedPath coerceToSingleDerivedPath(
std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(
const PosIdx pos,
Value & v,
std::string_view errorCtx,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings,
bool skipStringValidation = false);
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Coerce to `SingleDerivedPath`.
*
* Must be a string which is either a literal store path or a
* "placeholder (see `DownstreamPlaceholder`).
*
* Even more importantly, the string context must be exactly one
* element, which is either a `NixStringContextElem::Opaque` or
* `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep`
* is not permitted).
*
* The string is parsed based on the context --- the context is the
* source of truth, and ultimately tells us what we want, and then
* we ensure the string corresponds to it.
*/
SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
#if NIX_USE_BOEHMGC
/** A GC root for the baseEnv reference. */
@@ -1009,12 +1002,6 @@ public:
[[nodiscard]] StringMap
realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
/**
* Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`.
*/
SourcePath realisePath(
const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full);
/**
* Realise the given string with context, and return the string with outputs instead of downstream output
* placeholders.
+4 -20
View File
@@ -2,7 +2,6 @@
///@file
#include "nix/expr/eval.hh"
#include "nix/store/derived-path.hh"
#include "nix/store/path.hh"
#include <string>
@@ -21,23 +20,10 @@ public:
private:
EvalState * state;
struct Path {
/**
* Underlying semantic path, valid in CA and IA cases.
*/
SingleDerivedPath derivedPath;
/**
* If we don't need a placeholder, we'll have a real store path, which we can remember here.
* For CA derivations using placeholders, this will be std::nullopt.
*/
std::optional<StorePath> storePath;
};
mutable std::string name;
mutable std::string system;
mutable std::optional<std::optional<Path>> drvPath;
mutable std::optional<Path> outPath;
mutable std::optional<std::optional<StorePath>> drvPath;
mutable std::optional<StorePath> outPath;
mutable std::string outputName;
Outputs outputs;
@@ -66,10 +52,8 @@ public:
std::string queryName() const;
std::string querySystem() const;
std::optional<StorePath> queryDrvPath() const;
std::optional<Path> queryDrvPathFlexible() const;
StorePath requireDrvPath() const;
StorePath queryOutPath() const;
Path queryOutPathFlexible() const;
std::string queryOutputName() const;
/**
* Return the unordered map of output names to (optional) output paths.
@@ -97,12 +81,12 @@ public:
void setDrvPath(StorePath path)
{
drvPath = {{Path{SingleDerivedPath::Opaque{path}, std::move(path)}}};
drvPath = {{std::move(path)}};
}
void setOutPath(StorePath path)
{
outPath = Path{SingleDerivedPath::Opaque{path}, std::move(path)};
outPath = {{std::move(path)}};
}
void setFailed()
+2 -2
View File
@@ -533,8 +533,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF
* Packs discriminator bits into the pointer alignment niches.
*/
template<std::size_t ptrSize>
class alignas(16)
ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>> : public detail::ValueBase
class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>>
: public detail::ValueBase
{
/* Needs a dependent type name in order for member functions (and
* potentially ill-formed bit casts) to be SFINAE'd out.
+13 -36
View File
@@ -3,7 +3,7 @@
%define api.namespace { ::nix::parser }
%define api.parser.class { BisonParser }
%locations
%define parse.error detailed
%define parse.error verbose
%defines
/* %no-lines */
%parse-param { void * scanner }
@@ -140,41 +140,18 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%type <Expr *> path_start
%type <ToBeStringyExpr> string_parts string_attr
%type <StringToken> attr
%token <StringToken> ID "identifier"
%token <StringToken> STR "string"
%token <StringToken> IND_STR "indented string"
%token <NixInt> INT_LIT "integer"
%token <NixFloat> FLOAT_LIT "floating-point literal"
%token <StringToken> PATH "path"
%token <StringToken> HPATH "'~/…' path"
%token <StringToken> SPATH "'<…>' path"
%token <StringToken> PATH_END "end of path"
%token <StringToken> URI "URI"
%token IF "'if'"
%token THEN "'then'"
%token ELSE "'else'"
%token ASSERT "'assert'"
%token WITH "'with'"
%token LET "'let'"
%token IN_KW "'in'"
%token REC "'rec'"
%token INHERIT "'inherit'"
%token EQ "'=='"
%token NEQ "'!='"
%token LEQ "'<='"
%token GEQ "'>='"
%token UPDATE "'//'"
%token CONCAT "'++'"
%token AND "'&&'"
%token OR "'||'"
%token IMPL "'->'"
%token OR_KW "'or'"
%token PIPE_FROM "'<|'"
%token PIPE_INTO "'|>'"
%token DOLLAR_CURLY "'${'"
%token IND_STRING_OPEN "start of an indented string"
%token IND_STRING_CLOSE "end of an indented string"
%token ELLIPSIS "'...'"
%token <StringToken> ID
%token <StringToken> STR IND_STR
%token <NixInt> INT_LIT
%token <NixFloat> FLOAT_LIT
%token <StringToken> PATH HPATH SPATH PATH_END
%token <StringToken> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */
%token IND_STRING_OPEN IND_STRING_CLOSE
%token ELLIPSIS
%right IMPL
%left OR
+16 -12
View File
@@ -157,20 +157,24 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
return res;
}
SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks)
static SourcePath realisePath(
EvalState & state,
const PosIdx pos,
Value & v,
std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full)
{
NixStringContext context;
auto path = coerceToPath(noPos, v, context, "while realising the context of a path");
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try {
if (!context.empty() && path.accessor == rootFS) {
auto rewrites = realiseContext(context);
if (!context.empty() && path.accessor == state.rootFS) {
auto rewrites = state.realiseContext(context);
path = {path.accessor, CanonPath(rewriteStrings(path.path.abs(), rewrites))};
}
return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path;
} catch (Error & e) {
e.addTrace(positions[pos], "while realising the context of path '%s'", path);
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw;
}
}
@@ -290,7 +294,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
argument. */
static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
{
auto path = state.realisePath(pos, vPath, std::nullopt);
auto path = realisePath(state, pos, vPath, std::nullopt);
auto path2 = path.path.abs();
// FIXME
@@ -443,7 +447,7 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
/* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v)
{
auto path = state.realisePath(pos, *args[0]);
auto path = realisePath(state, pos, *args[0]);
std::string sym(
state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative"));
@@ -1967,7 +1971,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value ** args,
arg.type() == nString && (arg.string_view().ends_with("/") || arg.string_view().ends_with("/."));
auto symlinkResolution = mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors;
auto path = state.realisePath(pos, arg, symlinkResolution);
auto path = realisePath(state, pos, arg, symlinkResolution);
auto st = path.maybeLstat();
auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
@@ -2074,7 +2078,7 @@ static RegisterPrimOp primop_dirOf({
/* Return the contents of a file as a string. */
static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Value & v)
{
auto path = state.realisePath(pos, *args[0]);
auto path = realisePath(state, pos, *args[0]);
auto s = path.readFile();
if (s.find((char) 0) != std::string::npos)
state.error<EvalError>("the contents of the file '%1%' cannot be represented as a Nix string", path)
@@ -2309,7 +2313,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
if (!ha)
state.error<EvalError>("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow();
auto path = state.realisePath(pos, *args[1]);
auto path = realisePath(state, pos, *args[1]);
v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
}
@@ -2361,7 +2365,7 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
static void prim_readFileType(EvalState & state, const PosIdx pos, Value ** args, Value & v)
{
auto path = state.realisePath(pos, *args[0], std::nullopt);
auto path = realisePath(state, pos, *args[0], std::nullopt);
/* Retrieve the directory entry type and stringize it. */
v = fileTypeToString(state, path.lstat().type);
}
@@ -2379,7 +2383,7 @@ static RegisterPrimOp primop_readFileType({
/* Read a directory (without . or ..) */
static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Value & v)
{
auto path = state.realisePath(pos, *args[0]);
auto path = realisePath(state, pos, *args[0]);
// Retrieve directory entries for all nodes in a directory.
// This is similar to `getFileType` but is optimized to reduce system calls
+1 -1
View File
@@ -41,7 +41,7 @@ struct CacheImpl : Cache
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
+2 -2
View File
@@ -489,11 +489,11 @@ void InputScheme::clone(
const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
{
if (std::filesystem::exists(destDir))
throw Error("cannot clone into existing path %s", PathFmt(destDir));
throw Error("cannot clone into existing path %s", destDir);
auto [accessor, input2] = getAccessor(settings, store, input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), PathFmt(destDir)));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
RestoreSink sink(/*startFsync=*/false);
sink.dstPath = destDir;
@@ -10,6 +10,12 @@ std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(co
return next->getPhysicalPath(prefix / path);
}
std::string FilteringSourceAccessor::readFile(const CanonPath & path)
{
checkAccess(path);
return next->readFile(prefix / path);
}
void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
{
checkAccess(path);
+2 -2
View File
@@ -273,7 +273,7 @@ void Fetch::fetch(
+ "/" + pointer->oid;
std::filesystem::path cachePath = cacheDir / key;
if (pathExists(cachePath)) {
debug("using cache entry %s -> %s", key, PathFmt(cachePath));
debug("using cache entry %s -> %s", key, cachePath);
sink(readFile(cachePath));
return;
}
@@ -301,7 +301,7 @@ void Fetch::fetch(
sizeCallback(size);
downloadToSink(ourl, authHeader, sink, sha256, size);
debug("creating cache entry %s -> %s", key, PathFmt(cachePath));
debug("creating cache entry %s -> %s", key, cachePath);
if (!pathExists(cachePath.parent_path()))
createDirs(cachePath.parent_path());
writeFile(cachePath, sink.s);
+11 -19
View File
@@ -209,14 +209,14 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
return;
if (!options.create)
throw Error("Git repository %s does not exist.", PathFmt(path));
throw Error("Git repository %s does not exist.", path);
std::filesystem::path tmpDir = createTempDir(path.parent_path());
AutoDelete delTmpDir(tmpDir, true);
Repository tmpRepo;
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
throw Error("creating Git repository %s: %s", PathFmt(path), git_error_last()->message);
throw Error("creating Git repository %s: %s", path, git_error_last()->message);
try {
std::filesystem::rename(tmpDir, path);
} catch (std::filesystem::filesystem_error & e) {
@@ -226,7 +226,7 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
|| e.code() == std::errc::directory_not_empty) {
return;
} else
throw SysError("moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path));
throw SysError("moving temporary git repository from %s to %s", tmpDir, path);
}
// we successfully moved the repository, so the temporary directory no longer exists.
delTmpDir.cancel();
@@ -266,7 +266,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
initRepoAtomically(path, options);
if (git_repository_open(Setter(repo), path.string().c_str()))
throw Error("opening Git repository %s: %s", PathFmt(path), git_error_last()->message);
throw Error("opening Git repository %s: %s", path, git_error_last()->message);
ObjectDb odb;
if (options.packfilesOnly) {
@@ -766,7 +766,7 @@ struct GitSourceAccessor : SourceAccessor
{
}
void readBlob(const CanonPath & path, bool symlink, Sink & sink, std::function<void(uint64_t)> sizeCallback)
std::string readBlob(const CanonPath & path, bool symlink)
{
auto state(state_.lock());
@@ -785,22 +785,16 @@ struct GitSourceAccessor : SourceAccessor
e.addTrace({}, "while smudging git-lfs file '%s'", path);
throw;
}
sizeCallback(s.s.size());
StringSource source{s.s};
source.drainInto(sink);
return;
return s.s;
}
}
auto view = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
sizeCallback(view.size());
StringSource source{view};
source.drainInto(sink);
return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
}
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
std::string readFile(const CanonPath & path) override
{
return readBlob(path, false, sink, sizeCallback);
return readBlob(path, false);
}
bool pathExists(const CanonPath & path) override
@@ -867,9 +861,7 @@ struct GitSourceAccessor : SourceAccessor
std::string readLink(const CanonPath & path) override
{
StringSink s;
readBlob(path, true, s, [&](uint64_t size) { s.s.reserve(size); });
return std::move(s.s);
return readBlob(path, true);
}
/**
@@ -1433,7 +1425,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
try {
writeFull(fdTemp.get(), configS);
} catch (SystemError & e) {
} catch (SysError & e) {
e.addTrace({}, "while writing .gitmodules file to temporary file");
throw;
}
+12 -13
View File
@@ -37,7 +37,7 @@ namespace {
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
bool isCacheFileWithinTtl(time_t now, const PosixStat & st)
bool isCacheFileWithinTtl(time_t now, const struct stat & st)
{
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
}
@@ -72,10 +72,10 @@ std::optional<std::string> readHead(const std::filesystem::path & path)
if (const auto parseResult = git::parseLsRemoteLine(line); parseResult && parseResult->reference == "HEAD") {
switch (parseResult->kind) {
case git::LsRemoteRefLine::Kind::Symbolic:
debug("resolved HEAD ref '%s' for repo %s", parseResult->target, PathFmt(path));
debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path);
break;
case git::LsRemoteRefLine::Kind::Object:
debug("resolved HEAD rev '%s' for repo %s", parseResult->target, PathFmt(path));
debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path);
break;
}
return parseResult->target;
@@ -113,11 +113,11 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl, bool sh
std::filesystem::path headRefFile = cacheDir / "HEAD";
time_t now = time(0);
auto st = maybeStat(headRefFile);
struct stat st;
std::optional<std::string> cachedRef;
if (st) {
if (stat(headRefFile.string().c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, *st)) {
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
@@ -753,10 +753,9 @@ struct GitInputScheme : InputScheme
"\n"
"git -C %2% add \"%1%\"",
path.rel(),
PathFmt(repoPath));
repoPath);
else
return RestrictedPathError(
"Path '%s' does not exist in Git repository %s.", path.rel(), PathFmt(repoPath));
return RestrictedPathError("Path '%s' does not exist in Git repository %s.", path.rel(), repoPath);
};
}
@@ -819,10 +818,10 @@ struct GitInputScheme : InputScheme
if (getAllRefsAttr(input)) {
doFetch = true;
} else {
/* If the local ref is older than 'tarball-ttl' seconds, do a
/* If the local ref is older than ‘tarball-ttl’ seconds, do a
git fetch to update the local ref to the remote ref. */
auto st = maybeStat(localRefFile);
doFetch = !st || !isCacheFileWithinTtl(now, *st);
struct stat st;
doFetch = stat(localRefFile.string().c_str(), &st) != 0 || !isCacheFileWithinTtl(now, st);
}
}
@@ -849,7 +848,7 @@ struct GitInputScheme : InputScheme
if (!input.getRev())
setWriteTime(localRefFile, now, now);
} catch (Error & e) {
warn("could not update mtime for file %s: %s", PathFmt(localRefFile), e.info().msg);
warn("could not update mtime for file %s: %s", localRefFile, e.info().msg);
}
if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref))
warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
@@ -34,7 +34,7 @@ struct FilteringSourceAccessor : SourceAccessor
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
using SourceAccessor::readFile;
std::string readFile(const CanonPath & path) override;
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
+1 -1
View File
@@ -154,7 +154,7 @@ struct PathInputScheme : InputScheme
time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", PathFmt(absPath)));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: try to substitute storePath.
auto src = sinkToSource(
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
+4 -6
View File
@@ -121,15 +121,14 @@ static DownloadTarballResult downloadTarball_(
url);
}
if (!exists(localPath)) {
throw Error("tarball %s does not exist.", PathFmt(localPath));
throw Error("tarball '%s' does not exist.", localPath);
}
if (is_directory(localPath)) {
if (exists(localPath / ".git")) {
throw Error(
"tarball %s is a git repository, not a tarball. Please use `git+file` as the scheme.",
PathFmt(localPath));
"tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath);
}
throw Error("tarball %s is a directory, not a file.", PathFmt(localPath));
throw Error("tarball '%s' is a directory, not a file.", localPath);
}
}
@@ -177,8 +176,7 @@ static DownloadTarballResult downloadTarball_(
the entire file to disk so libarchive can access it
in random-access mode. */
auto [fdTemp, path] = createTempFile("nix-zipfile");
cleanupTemp.cancel();
cleanupTemp = {path};
cleanupTemp.reset(path);
debug("downloading '%s' into '%s'...", url, path);
{
FdSink sink(fdTemp.get());
+3 -3
View File
@@ -850,11 +850,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
auto s = chomp(diff);
if (lockFileExists) {
if (s.empty())
warn("updating lock file %s", PathFmt(outputLockFilePath));
warn("updating lock file %s", outputLockFilePath);
else
warn("updating lock file %s:\n%s", PathFmt(outputLockFilePath), s);
warn("updating lock file %s:\n%s", outputLockFilePath, s);
} else
warn("creating lock file %s: \n%s", PathFmt(outputLockFilePath), s);
warn("creating lock file %s: \n%s", outputLockFilePath, s);
std::optional<std::string> commitMessage = std::nullopt;
+4 -4
View File
@@ -83,8 +83,8 @@ void initPlugins()
checkInterrupt();
pluginFiles.emplace_back(ent.path());
}
} catch (SystemError & e) {
if (!e.is(std::errc::not_a_directory))
} catch (SysError & e) {
if (e.errNo != ENOTDIR)
throw;
pluginFiles.emplace_back(pluginFile);
}
@@ -95,7 +95,7 @@ void initPlugins()
#ifndef _WIN32 // TODO implement via DLL loading on Windows
void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
if (!handle)
throw Error("could not dynamically open plugin file %s: %s", PathFmt(file), dlerror());
throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
/* Older plugins use a statically initialized object to run their code.
Newer plugins can also export nix_plugin_entry() */
@@ -103,7 +103,7 @@ void initPlugins()
if (nix_plugin_entry)
nix_plugin_entry();
#else
throw Error("could not dynamically open plugin file %s", PathFmt(file));
throw Error("could not dynamically open plugin file '%s'", file);
#endif
}
}
+2 -1
View File
@@ -304,10 +304,11 @@ void printVersion(const std::string & programName)
std::cout << "System type: " << settings.thisSystem << "\n";
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf").string() << "\n";
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf") << "\n";
std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n";
std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n";
std::cout << "Data directory: " << settings.nixDataDir << "\n";
}
throw Exit();
}
-16
View File
@@ -338,20 +338,4 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
NIXC_CATCH_ERRS_NULL
}
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash)
{
if (context)
context->last_err_code = NIX_OK;
try {
std::optional<nix::StorePath> s = store->ptr->queryPathFromHashPart(hash);
if (!s.has_value()) {
return nullptr;
}
return new StorePath{std::move(s.value())};
}
NIXC_CATCH_ERRS_NULL
}
} // extern "C"
-11
View File
@@ -259,17 +259,6 @@ nix_err nix_store_get_fs_closure(
*/
nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path);
/**
* @brief Query the full store path given the hash part of a valid store
* path, or empty if no matching path is found.
*
* @param[out] context Optional, stores error information
* @param[in] store nix store reference
* @param[in] hash Hash part of path as a string
* @return Store path reference, NULL if no matching path is found.
*/
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash);
// cffi end
#ifdef __cplusplus
}
-125
View File
@@ -1,125 +0,0 @@
#include "nix/store/tests/https-store.hh"
#include <thread>
namespace nix::testing {
void TestHttpBinaryCacheStore::init()
{
BinaryCacheStore::init();
}
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore() const
{
auto store = make_ref<TestHttpBinaryCacheStore>(
ref{// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())});
store->init();
return store;
}
void HttpsBinaryCacheStoreTest::openssl(Strings args)
{
runProgram("openssl", /*lookupPath=*/true, args);
}
void HttpsBinaryCacheStoreTest::SetUp()
{
LibStoreNetworkTest::SetUp();
#ifdef _WIN32
GTEST_SKIP() << "HTTPS store tests are not supported on Windows";
#endif
tmpDir = createTempDir();
cacheDir = tmpDir / "cache";
delTmpDir = std::make_unique<AutoDelete>(tmpDir);
localCacheStore =
make_ref<LocalBinaryCacheStoreConfig>("file", cacheDir.string(), LocalBinaryCacheStoreConfig::Params{})
->openStore();
caCert = tmpDir / "ca.crt";
caKey = tmpDir / "ca.key";
serverCert = tmpDir / "server.crt";
serverKey = tmpDir / "server.key";
clientCert = tmpDir / "client.crt";
clientKey = tmpDir / "client.key";
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost"});
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1"});
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
// clang-format on
#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */
auto args = serverArgs();
serverPid = startProcess(
[&] {
if (chdir(cacheDir.c_str()) == -1)
_exit(1);
std::vector<char *> argv;
argv.push_back(const_cast<char *>("openssl"));
for (auto & a : args)
argv.push_back(const_cast<char *>(a.c_str()));
argv.push_back(nullptr);
execvp("openssl", argv.data());
_exit(1);
},
{.dieWithParent = true});
#endif
/* As an optimization, sleep for a bit to allow the server to come up to avoid retrying when connecting.
This won't make the tests fail, but does make them run faster. We don't need to overcomplicate by waiting
for the port explicitly - this is enough. */
std::this_thread::sleep_for(std::chrono::milliseconds(50));
/* FIXME: Don't use global settings. Tests are not run concurrently, so this is fine for now. */
oldCaCert = settings.caFile;
settings.caFile = caCert.string();
}
void HttpsBinaryCacheStoreTest::TearDown()
{
settings.caFile = oldCaCert;
serverPid.kill();
delTmpDir.reset();
}
std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
{
return {
"s_server",
"-accept",
std::to_string(port),
"-cert",
serverCert.string(),
"-key",
serverKey.string(),
"-WWW", /* Serve from current directory. */
"-quiet",
};
}
std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
{
auto args = HttpsBinaryCacheStoreTest::serverArgs();
/* With the -Verify option the client must supply a certificate or an error occurs, which is not the
case with -verify. */
args.insert(args.end(), {"-CAfile", caCert.string(), "-Verify", "1", "-verify_return_error"});
return args;
}
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig(BinaryCacheStoreConfig::Params params)
{
auto res = make_ref<TestHttpBinaryCacheStoreConfig>("https", fmt("localhost:%d", port), std::move(params));
res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
return res;
}
} // namespace nix::testing
@@ -1,93 +0,0 @@
#pragma once
///@file
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "nix/store/tests/libstore-network.hh"
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/store-api.hh"
#include "nix/store/globals.hh"
#include "nix/store/local-binary-cache-store.hh"
#include "nix/util/file-system.hh"
#include "nix/util/processes.hh"
namespace nix::testing {
class TestHttpBinaryCacheStoreConfig;
/**
* Test shim for testing. We don't want to use the on-disk narinfo cache in unit
* tests.
*/
class TestHttpBinaryCacheStore : public HttpBinaryCacheStore
{
public:
TestHttpBinaryCacheStore(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore(config)
{
diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
}
void init() override;
};
class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
{
public:
TestHttpBinaryCacheStoreConfig(
std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params)
: StoreConfig(params)
, HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
{
}
ref<TestHttpBinaryCacheStore> openTestStore() const;
ref<Store> openStore() const override
{
return openTestStore();
}
};
class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
{
std::unique_ptr<AutoDelete> delTmpDir;
public:
static void SetUpTestSuite()
{
initLibStore(/*loadConfig=*/false);
}
protected:
std::filesystem::path tmpDir, cacheDir;
std::filesystem::path caCert, caKey, serverCert, serverKey;
std::filesystem::path clientCert, clientKey;
std::string oldCaCert;
Pid serverPid;
uint16_t port = 8443;
std::shared_ptr<Store> localCacheStore;
static void openssl(Strings args);
void SetUp() override;
void TearDown() override;
virtual std::vector<std::string> serverArgs();
ref<TestHttpBinaryCacheStoreConfig> makeConfig(BinaryCacheStoreConfig::Params params);
};
class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
{
protected:
std::vector<std::string> serverArgs() override;
};
} // namespace nix::testing
@@ -1,39 +0,0 @@
#pragma once
/// @file
#include <gtest/gtest.h>
namespace nix::testing {
/**
* Whether to run network tests. This is global so that the test harness can
* enable this by default if we can run tests in isolation.
*/
extern bool networkTestsAvailable;
/**
* Set up network tests and, if on linux, create a new network namespace for
* tests with a loopback interface. This is to avoid binding to ports in the
* host's namespace.
*/
void setupNetworkTests();
class LibStoreNetworkTest : public virtual ::testing::Test
{
protected:
void SetUp() override
{
if (networkTestsAvailable)
return;
static bool warned = false;
if (!warned) {
warned = true;
GTEST_SKIP()
<< "Network tests not enabled by default without user namespaces, use NIX_TEST_FORCE_NETWORK_TESTS=1 to override";
} else {
GTEST_SKIP();
}
}
};
} // namespace nix::testing
@@ -4,8 +4,6 @@ include_dirs = [ include_directories('../../..') ]
headers = files(
'derived-path.hh',
'https-store.hh',
'libstore-network.hh',
'libstore.hh',
'nix_api_store.hh',
'outputs-spec.hh',
@@ -1,60 +0,0 @@
#include "nix/store/tests/libstore-network.hh"
#include "nix/util/error.hh"
#include "nix/util/environment-variables.hh"
#ifdef __linux__
# include "nix/util/file-system.hh"
# include "nix/util/linux-namespaces.hh"
# include <sched.h>
# include <sys/ioctl.h>
# include <net/if.h>
# include <netinet/in.h>
#endif
namespace nix::testing {
bool networkTestsAvailable = false;
#ifdef __linux__
static void enterNetworkNamespace()
{
auto uid = ::getuid();
auto gid = ::getgid();
if (::unshare(CLONE_NEWUSER | CLONE_NEWNET) == -1)
throw SysError("setting up a private network namespace for tests");
std::filesystem::path procSelf = "/proc/self";
writeFile(procSelf / "setgroups", "deny");
writeFile(procSelf / "uid_map", fmt("%d %d 1", uid, uid));
writeFile(procSelf / "gid_map", fmt("%d %d 1", gid, gid));
AutoCloseFD fd(::socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
if (!fd)
throw SysError("cannot open IP socket for loopback interface");
struct ::ifreq ifr = {};
strcpy(ifr.ifr_name, "lo");
ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
throw SysError("cannot set loopback interface flags");
}
#endif
void setupNetworkTests()
try {
networkTestsAvailable = getEnvOs(OS_STR("NIX_TEST_FORCE_NETWORK_TESTS")).has_value();
#ifdef __linux__
if (!networkTestsAvailable && userNamespacesSupported()) {
enterNetworkNamespace();
networkTestsAvailable = true;
}
#endif
} catch (SystemError & e) {
/* Ignore any set up errors. */
}
} // namespace nix::testing
-5
View File
@@ -28,15 +28,10 @@ subdir('nix-meson-build-support/subprojects')
rapidcheck = dependency('rapidcheck')
deps_public += rapidcheck
gtest = dependency('gtest')
deps_public += gtest
subdir('nix-meson-build-support/common')
sources = files(
'derived-path.cc',
'https-store.cc',
'libstore-network.cc',
'outputs-spec.cc',
'path.cc',
'test-main.cc',
-2
View File
@@ -7,7 +7,6 @@
nix-store-c,
rapidcheck,
gtest,
# Configuration Options
@@ -40,7 +39,6 @@ mkMesonLibrary (finalAttrs: {
nix-store
nix-store-c
rapidcheck
gtest
];
mesonFlags = [
+2 -10
View File
@@ -138,11 +138,7 @@ CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures =
{
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -158,11 +154,7 @@ READ_CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures =
{
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -6,8 +6,8 @@
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
]
}
]
@@ -10,8 +10,8 @@
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
]
}
]
@@ -12,8 +12,8 @@
],
"registrationTime": 23423,
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
],
"storeDir": "/nix/store",
"ultimate": true,
@@ -15,8 +15,8 @@
],
"registrationTime": 23423,
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
],
"storeDir": "/nix/store",
"ultimate": true,
@@ -9,8 +9,8 @@
],
"registrationTime": 23423,
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
],
"storeDir": "/nix/store",
"ultimate": true,
@@ -12,8 +12,8 @@
],
"registrationTime": 23423,
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
],
"storeDir": "/nix/store",
"ultimate": true,
@@ -3,6 +3,6 @@
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdfasdfasdf"
]
}
@@ -6,8 +6,8 @@
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
]
}
]
@@ -10,8 +10,8 @@
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
]
}
]
@@ -27,8 +27,8 @@
],
"registrationTime": null,
"signatures": [
"fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"fake-sig-1",
"fake-sig-2"
],
"storeDir": "/nix/store",
"ultimate": false,
@@ -6,8 +6,8 @@
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
]
}
]
@@ -10,8 +10,8 @@
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
"signatures": [
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"asdf",
"qwer"
]
}
]
@@ -24,8 +24,8 @@
],
"registrationTime": 23423,
"signatures": [
"fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
"fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
"fake-sig-1",
"fake-sig-2"
],
"storeDir": "/nix/store",
"ultimate": false,
@@ -1,9 +1,6 @@
#include <gtest/gtest.h>
#include <regex>
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/tests/https-store.hh"
#include "nix/util/fs-sink.hh"
namespace nix {
@@ -37,88 +34,4 @@ TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams)
EXPECT_EQ(config.getReference().params, params);
}
using testing::HttpsBinaryCacheStoreMtlsTest;
using testing::HttpsBinaryCacheStoreTest;
using namespace std::string_view_literals;
using namespace std::string_literals;
TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo)
{
auto config = makeConfig({});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
auto withNoRetries()
{
auto oldTries = fileTransferSettings.tries.get();
Finally restoreTries = [=]() { fileTransferSettings.tries = oldTries; };
fileTransferSettings.tries = 1; /* FIXME: Don't use global settings. */
return restoreTries;
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
{
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
EXPECT_NO_THROW(store->queryPathInfo(path));
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert)
{
auto restoreTries = withNoRetries();
auto config = makeConfig({});
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
{
auto wrongKey = tmpDir / "wrong.key";
auto wrongCert = tmpDir / "wrong.crt";
// clang-format off
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", wrongKey.string()});
openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"});
// clang-format on
auto config = makeConfig({
{"tls-certificate"s, wrongCert.string()},
{"tls-private-key"s, wrongKey.string()},
});
auto restoreTries = withNoRetries();
EXPECT_THROW(config->openStore(), Error);
}
TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority)
{
StringSource dump{"test"sv};
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
for (auto & entry : DirectoryIterator{cacheDir})
if (entry.path().extension() == ".narinfo") {
auto content = readFile(entry.path());
content = std::regex_replace(content, std::regex("URL: nar/"), fmt("URL: https://127.0.0.1:%d/nar/", port));
writeFile(entry.path(), content);
}
auto config = makeConfig({
{"tls-certificate"s, clientCert.string()},
{"tls-private-key"s, clientKey.string()},
});
auto store = config->openStore();
auto restoreTries = withNoRetries();
auto info = store->queryPathInfo(path);
NullSink null;
EXPECT_THROW(store->narFromPath(path, null), Error);
}
} // namespace nix
-2
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/tests/test-main.hh"
#include "nix/store/tests/libstore-network.hh"
using namespace nix;
@@ -11,7 +10,6 @@ int main(int argc, char ** argv)
if (res)
return res;
nix::testing::setupNetworkTests();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
+1 -2
View File
@@ -2,7 +2,6 @@
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "nix/store/globals.hh"
#include "nix/store/sqlite.hh"
#include <sqlite3.h>
@@ -49,7 +48,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
// We're going to pay special attention to the id field because we had a bug
// that changed it.
db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
db = SQLite(dbPath);
getIds.create(db, "select id from BinaryCaches where url = 'http://foo'");
{
+1 -4
View File
@@ -59,10 +59,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo)
};
info.registrationTime = 23423;
info.ultimate = true;
info.sigs = {
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
};
info.sigs = {"asdf", "qwer"};
info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz";
info.compression = "xz";
+1 -5
View File
@@ -9,7 +9,6 @@
nix-store-c,
nix-store-test-support,
sqlite,
openssl,
rapidcheck,
gtest,
@@ -76,10 +75,7 @@ mkMesonExecutable (finalAttrs: {
runCommand "${finalAttrs.pname}-run"
{
meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
nativeBuildInputs = [
writableTmpDirAsHomeHook
openssl
];
buildInputs = [ writableTmpDirAsHomeHook ];
}
(
''
+1 -4
View File
@@ -66,10 +66,7 @@ static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo)
};
info.registrationTime = 23423;
info.ultimate = true;
info.sigs = {
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
};
info.sigs = {"asdf", "qwer"};
}
return info;
}
+1 -3
View File
@@ -70,9 +70,7 @@ INSTANTIATE_TEST_SUITE_P(
[&] {
auto r = simple;
// FIXME actually sign properly
r.signatures = {
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
};
r.signatures = {"asdfasdfasdf"};
return r;
}(),
}));
+4 -12
View File
@@ -109,11 +109,7 @@ VERSIONED_CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures =
{
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -131,11 +127,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures =
{
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -329,8 +321,8 @@ VERSIONED_CHARACTERIZATION_TEST(
info.narSize = 34878;
info.sigs =
{
Signature{.keyName = "fake-sig-1", .sig = std::string(64, '\0')},
Signature{.keyName = "fake-sig-2", .sig = std::string(64, '\0')},
"fake-sig-1",
"fake-sig-2",
},
static_cast<UnkeyedValidPathInfo>(std::move(info));
}),
+4 -12
View File
@@ -162,11 +162,7 @@ VERSIONED_CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures =
{
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -184,11 +180,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
Realisation{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures =
{
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
},
.signatures = {"asdf", "qwer"},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -549,8 +541,8 @@ VERSIONED_CHARACTERIZATION_TEST(
info.narSize = 34878;
info.sigs =
{
Signature{.keyName = "fake-sig-1", .sig = std::string(64, '\0')},
Signature{.keyName = "fake-sig-2", .sig = std::string(64, '\0')},
"fake-sig-1",
"fake-sig-2",
},
info;
}),
+12 -103
View File
@@ -13,11 +13,6 @@
// C library headers for SSO provider support
# include <aws/auth/credentials.h>
// C library headers for custom logging
# include <aws/common/logging.h>
# include <cstdarg>
# include <boost/unordered/concurrent_flat_map.hpp>
# include <chrono>
@@ -35,101 +30,6 @@ AwsAuthError::AwsAuthError(int errorCode)
namespace {
/**
* Map AWS log level to Nix verbosity.
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
*
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
* considers "info" includes low-level details like "Initializing epoll" and
* "Starting event-loop thread". What it considers "errors" includes expected
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
*
* To avoid spamming users, we only show FATAL at default verbosity. Everything
* else requires -vvvvv (lvlDebug) or higher to see.
*/
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
{
switch (level) {
case AWS_LL_FATAL:
return lvlError;
case AWS_LL_ERROR:
case AWS_LL_WARN:
case AWS_LL_INFO:
return lvlDebug;
case AWS_LL_DEBUG:
case AWS_LL_TRACE:
return lvlVomit;
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
case AWS_LL_NONE:
case AWS_LL_COUNT:
return lvlDebug;
}
unreachable();
}
/**
* Custom AWS logger that routes logs through Nix's logging infrastructure.
*
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
* or filename-based logging. The underlying C library supports custom loggers
* via aws_logger struct with a vtable containing callback functions.
*/
static int nixAwsLoggerLog(
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
{
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
if (nixLevel > verbosity)
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
va_list args;
va_start(args, format);
std::array<char, 4096> buffer{};
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
va_end(args);
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
return AWS_OP_SUCCESS;
const char * subjectName = aws_log_subject_name(subject);
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
return AWS_OP_SUCCESS;
}
/**
* Get current log level for a subject - determines which messages will be logged.
* Must be consistent with awsLogLevelToVerbosity mapping.
*/
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
{
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
if (verbosity >= lvlVomit)
return AWS_LL_TRACE;
if (verbosity >= lvlDebug)
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
return AWS_LL_FATAL;
}
static void initialiseAwsLogger()
{
static std::once_flag initialised; /* aws_logger_set must only be called once */
std::call_once(initialised, []() {
static aws_logger_vtable nixAwsLoggerVtable = {
.log = nixAwsLoggerLog,
.get_log_level = nixAwsLoggerGetLevel,
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
.set_log_level = nullptr,
};
static aws_logger nixAwsLogger = {
.vtable = &nixAwsLoggerVtable,
.allocator = nullptr,
.p_impl = nullptr,
};
aws_logger_set(&nixAwsLogger);
});
}
/**
* Helper function to wrap a C credentials provider in the C++ interface.
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
@@ -219,9 +119,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
public:
AwsCredentialProviderImpl()
{
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
initialiseAwsLogger();
// Map Nix's verbosity to AWS CRT log level
Aws::Crt::LogLevel logLevel;
if (verbosity >= lvlVomit) {
logLevel = Aws::Crt::LogLevel::Trace;
} else if (verbosity >= lvlDebug) {
logLevel = Aws::Crt::LogLevel::Debug;
} else if (verbosity >= lvlChatty) {
logLevel = Aws::Crt::LogLevel::Info;
} else {
logLevel = Aws::Crt::LogLevel::Warn;
}
apiHandle.InitializeLogging(logLevel, stderr);
// Create a shared TLS context for SSO (required for HTTPS connections)
auto allocator = Aws::Crt::ApiAllocator();
+13 -11
View File
@@ -13,6 +13,7 @@
#include "nix/util/callback.hh"
#include "nix/util/signals.hh"
#include "nix/util/archive.hh"
#include "nix/store/nar-cache.hh"
#include <chrono>
#include <future>
@@ -26,6 +27,7 @@ namespace nix {
BinaryCacheStore::BinaryCacheStore(Config & config)
: config{config}
, narCache{config.localNarCache.get() ? std::make_shared<NarCache>(*config.localNarCache.get()) : nullptr}
{
if (config.secretKeyFile != "")
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
@@ -156,7 +158,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel);
TeeSink teeSinkUncompressed{*compressionSink, narHashSink};
TeeSource teeSource{narSource, teeSinkUncompressed};
narAccessor = makeNarAccessor(parseNarListing(teeSource));
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
fileSink.flush();
}
@@ -165,18 +167,18 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
auto info = mkInfo(narHashSink.finish());
auto narInfo = make_ref<NarInfo>(info);
narInfo->compression = config.compression.to_string(); // FIXME: Make NarInfo use CompressionAlgo
narInfo->compression = config.compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
+ (config.compression == CompressionAlgo::xz ? ".xz"
: config.compression == CompressionAlgo::bzip2 ? ".bz2"
: config.compression == CompressionAlgo::zstd ? ".zst"
: config.compression == CompressionAlgo::lzip ? ".lzip"
: config.compression == CompressionAlgo::lz4 ? ".lz4"
: config.compression == CompressionAlgo::brotli ? ".br"
: "");
+ (config.compression == "xz" ? ".xz"
: config.compression == "bzip2" ? ".bz2"
: config.compression == "zstd" ? ".zst"
: config.compression == "lzip" ? ".lzip"
: config.compression == "lz4" ? ".lz4"
: config.compression == "br" ? ".br"
: "");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(
@@ -552,7 +554,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
}
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
@@ -565,7 +567,7 @@ std::shared_ptr<SourceAccessor> BinaryCacheStore::getFSAccessor(const StorePath
return getRemoteFSAccessor(requireValidPath)->accessObject(storePath);
}
void BinaryCacheStore::addSignatures(const StorePath & storePath, const std::set<Signature> & sigs)
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
{
/* Note: this is inherently racy since there is no locking on
binary caches. In particular, with S3 this unreliable, even
-5
View File
@@ -75,11 +75,6 @@ static BuildResult::Failure::Status failureStatusFromString(std::string_view str
throw Error("unknown built result failure status '%s'", str);
}
[[noreturn]] void BuildResult::Failure::rethrow() const
{
throw BuildError(status, "%s", errorMsg);
}
} // namespace nix
namespace nlohmann {
+40 -39
View File
@@ -175,7 +175,7 @@ struct LogFile
AutoCloseFD fd;
std::shared_ptr<BufferedSink> fileSink, sink;
LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings);
LogFile(Store & store, const StorePath & drvPath);
~LogFile();
};
@@ -396,22 +396,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
if (useHook) {
co_return buildWithHook(
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
} else if (auto * localStoreP = dynamic_cast<LocalStore *>(&worker.store)) {
} else {
co_return buildLocally(
*localStoreP,
std::move(inputPaths),
std::move(initialOutputs),
std::move(drvOptions),
std::move(outputLocks),
externalBuilder);
} else {
throw Error(
R"(
Unable to build with a primary store that isn't a local store;
either pass a different '--store' or enable remote builds.
For more information check 'man nix.conf' and search for '/machines'.
)");
}
}
@@ -460,7 +451,7 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
hook->toHook.writeSide.close();
/* Create the log file and pipe. */
std::unique_ptr<LogFile> logFile = std::make_unique<LogFile>(worker.store, drvPath, settings.getLogFileSettings());
std::unique_ptr<LogFile> logFile = std::make_unique<LogFile>(worker.store, drvPath);
std::set<MuxablePipePollState::CommChannel> fds;
fds.insert(hook->fromHook.readSide.get());
@@ -620,7 +611,6 @@ Goal::Co DerivationBuildingGoal::buildWithHook(
}
Goal::Co DerivationBuildingGoal::buildLocally(
LocalStore & localStore,
StorePathSet inputPaths,
std::map<std::string, InitialOutput> initialOutputs,
DerivationOptions<StorePath> drvOptions,
@@ -629,15 +619,23 @@ Goal::Co DerivationBuildingGoal::buildLocally(
{
co_await yield();
if (!dynamic_cast<LocalStore *>(&worker.store)) {
throw Error(
R"(
Unable to build with a primary store that isn't a local store;
either pass a different '--store' or enable remote builds.
For more information check 'man nix.conf' and search for '/machines'.
)");
}
#ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows
throw UnimplementedError("building derivations is not yet implemented on Windows");
#else
std::unique_ptr<BuildLog> buildLog;
std::unique_ptr<LogFile> logFile;
auto openLogFile = [&]() {
logFile = std::make_unique<LogFile>(worker.store, drvPath, settings.getLogFileSettings());
};
auto openLogFile = [&]() { logFile = std::make_unique<LogFile>(worker.store, drvPath); };
auto closeLogFile = [&]() { logFile.reset(); };
@@ -656,7 +654,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
};
std::unique_ptr<Activity> actLock;
DerivationBuilderUnique builder;
std::unique_ptr<DerivationBuilder> builder;
Descriptor builderOut;
// Will continue here while waiting for a build user below
@@ -708,6 +706,9 @@ Goal::Co DerivationBuildingGoal::buildLocally(
}
};
auto * localStoreP = dynamic_cast<LocalStore *>(&worker.store);
assert(localStoreP);
decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = settings.sandboxPaths.get();
DesugaredEnv desugaredEnv;
@@ -720,7 +721,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
worker.store.toStorePath(i.second.source.string()).first, closure);
} catch (InvalidPath & e) {
} catch (Error & e) {
e.addTrace({}, "while processing sandbox path %s", PathFmt(i.second.source));
e.addTrace({}, "while processing sandbox path '%s'", i.second.source);
throw;
}
for (auto & i : closure) {
@@ -753,12 +754,12 @@ Goal::Co DerivationBuildingGoal::buildLocally(
already be created, so we don't need to create it again. */
builder = externalBuilder
? makeExternalDerivationBuilder(
localStore,
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
std::move(params),
*externalBuilder)
: makeDerivationBuilder(
localStore,
*localStoreP,
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
std::move(params));
}
@@ -1027,8 +1028,8 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
else if (reply != "accept")
throw Error("bad hook reply '%s'", reply);
} catch (SystemError & e) {
if (e.is(std::errc::broken_pipe)) {
} catch (SysError & e) {
if (e.errNo == EPIPE) {
printError("build hook died unexpectedly: %s", chomp(drainFD(worker.hook->fromHook.readSide.get())));
worker.hook = 0;
return rpDecline;
@@ -1040,9 +1041,9 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
#endif
}
LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings)
LogFile::LogFile(Store & store, const StorePath & drvPath)
{
if (!logSettings.keepLog)
if (!settings.keepLog)
return;
auto baseName = std::string(baseNameOf(store.printStorePath(drvPath)));
@@ -1051,11 +1052,11 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
if (auto localStore = dynamic_cast<LocalStore *>(&store))
logDir = localStore->config->logDir;
else
logDir = logSettings.nixLogDir;
logDir = settings.nixLogDir;
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
createDirs(dir);
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), logSettings.compressLog ? ".bz2" : "");
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : "");
fd = toDescriptor(open(
logFileName.c_str(),
@@ -1070,7 +1071,7 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
fileSink = std::make_shared<FdSink>(fd.get());
if (logSettings.compressLog)
if (settings.compressLog)
sink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *fileSink));
else
sink = fileSink;
@@ -1183,6 +1184,11 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs)
{
buildResult.inner = BuildResult::Success{
.status = status,
.builtOutputs = std::move(builtOutputs),
};
mcRunningBuilds.reset();
if (status == BuildResult::Success::Built)
@@ -1190,15 +1196,16 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status stat
worker.updateProgress();
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
.builtOutputs = std::move(builtOutputs),
});
return amDone(ecSuccess, std::nullopt);
}
Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
{
buildResult.inner = BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
};
mcRunningBuilds.reset();
if (ex.status == BuildResult::Failure::TimedOut)
@@ -1210,13 +1217,7 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
worker.updateProgress();
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
return amDone(ecFailed, {std::move(ex)});
}
} // namespace nix
+21 -21
View File
@@ -465,6 +465,20 @@ UnkeyedRealisation DerivationGoal::assertPathValidity()
Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, UnkeyedRealisation builtOutput)
{
buildResult.inner = BuildResult::Success{
.status = status,
.builtOutputs = {{
wantedOutput,
{
std::move(builtOutput),
DrvOutput{
.drvHash = outputHash,
.outputName = wantedOutput,
},
},
}},
};
mcExpectedBuilds.reset();
if (status == BuildResult::Success::Built)
@@ -472,24 +486,16 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
worker.updateProgress();
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
.builtOutputs = {{
wantedOutput,
{
std::move(builtOutput),
DrvOutput{
.drvHash = outputHash,
.outputName = wantedOutput,
},
},
}},
});
return amDone(ecSuccess, std::nullopt);
}
Goal::Done DerivationGoal::doneFailure(BuildError ex)
{
buildResult.inner = BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
};
mcExpectedBuilds.reset();
if (ex.status == BuildResult::Failure::TimedOut)
@@ -501,13 +507,7 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
worker.updateProgress();
return Goal::doneFailure(
ecFailed,
BuildResult::Failure{
.status = ex.status,
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
},
std::move(ex));
return amDone(ecFailed, {std::move(ex)});
}
} // namespace nix
@@ -98,13 +98,7 @@ Goal::Co DerivationTrampolineGoal::init()
trace("outer load and build derivation");
if (nrFailed != 0) {
co_return doneFailure(
ecFailed,
BuildResult::Failure{
.status = BuildResult::Failure::DependencyFailed,
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
},
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
}
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
+1 -7
View File
@@ -63,18 +63,12 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
std::vector<KeyedBuildResult> results;
results.reserve(state.size());
for (auto & [req, goalPtr] : state) {
/* Goals that were never started or were cancelled have exitCode
ecBusy and a default buildResult with empty errorMsg. Skip them
to avoid reporting spurious failures with empty messages. */
if (goalPtr->exitCode == Goal::ecBusy)
continue;
for (auto & [req, goalPtr] : state)
results.emplace_back(
KeyedBuildResult{
goalPtr->buildResult,
/* .path = */ req,
});
}
return results;
}
-13
View File
@@ -183,19 +183,6 @@ Co Goal::await(Goals new_waitees)
co_return Return{};
}
Goal::Done Goal::doneSuccess(BuildResult::Success success)
{
buildResult.inner = std::move(success);
return amDone(ecSuccess);
}
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
{
assert(result == ecFailed || result == ecNoSubstituters);
buildResult.inner = std::move(failure);
return amDone(result, std::move(ex));
}
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
{
trace("done");
+9 -10
View File
@@ -28,21 +28,20 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
{
return Goal::doneSuccess(
BuildResult::Success{
.status = status,
});
buildResult.inner = BuildResult::Success{
.status = status,
};
return amDone(ecSuccess);
}
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
{
debug(errorMsg);
return Goal::doneFailure(
result,
BuildResult::Failure{
.status = status,
.errorMsg = std::move(errorMsg),
});
buildResult.inner = BuildResult::Failure{
.status = status,
.errorMsg = std::move(errorMsg),
};
return amDone(result);
}
Goal::Co PathSubstitutionGoal::init()
+2 -6
View File
@@ -408,12 +408,8 @@ void Worker::waitForInput()
is a build timeout, then wait for input until the first
deadline for any child. */
auto nearest = steady_time_point::max(); // nearest deadline
auto localStore = dynamic_cast<LocalStore *>(&store);
if (localStore && localStore->config->getGCSettings().minFree.get() != 0)
// If we have a local store (and thus are capable of automatically collecting garbage) and configured to do so,
// periodically wake up to see if we need to run the garbage collector. (See the `autoGC` call site above in
// this file, also gated on having a local store. when we wake up, we intended to reach that call site.)
if (settings.minFree.get() != 0)
// Periodicallty wake up to see if we need to run the garbage collector.
nearest = before + std::chrono::seconds(10);
for (auto & i : children) {
if (!i.respectTimeouts)
+14 -9
View File
@@ -33,8 +33,8 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
try {
srcFiles = DirectoryIterator{srcDir};
} catch (SystemError & e) {
if (e.is(std::errc::not_a_directory)) {
} catch (SysError & e) {
if (e.errNo == ENOTDIR) {
warn("not including '%s' in the user environment because it's not a directory", srcDir);
return;
}
@@ -50,12 +50,17 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
auto srcFile = (std::filesystem::path{srcDir} / name).string();
auto dstFile = (std::filesystem::path{dstDir} / name).string();
auto srcStOpt = maybeStat(srcFile.c_str());
if (!srcStOpt) {
warn("skipping dangling symlink '%s'", dstFile);
continue;
struct stat srcSt;
try {
if (stat(srcFile.c_str(), &srcSt) == -1)
throw SysError("getting status of '%1%'", srcFile);
} catch (SysError & e) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
warn("skipping dangling symlink '%s'", dstFile);
continue;
}
throw;
}
auto & srcSt = *srcStOpt;
/* The files below are special-cased to that they don't show
* up in user profiles, either because they are useless, or
@@ -136,8 +141,8 @@ void buildProfile(const Path & out, Packages && pkgs)
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
} catch (SystemError & e) {
if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::not_a_directory))
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR)
throw;
}
};
+2 -2
View File
@@ -4,7 +4,6 @@
#include "nix/store/globals.hh"
#include "nix/util/archive.hh"
#include "nix/util/compression.hh"
#include "nix/util/file-system.hh"
namespace nix {
@@ -66,7 +65,8 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
auto executable = ctx.drv.env.find("executable");
if (executable != ctx.drv.env.end() && executable->second == "1") {
chmod(storePath, 0755);
if (chmod(storePath.c_str(), 0755) == -1)
throw SysError("making '%1%' executable", storePath);
}
};
Loaded 100 of 318 files, more files were not shown because too many files have changed in this diff. Show more