Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
545f5fb309 | ||
|
|
6312a94839 | ||
|
|
e77306792d | ||
|
|
24054b9c5d | ||
|
|
404cba4347 | ||
|
|
63aa29c2be | ||
|
|
3245211b29 | ||
|
|
ebb4271c6f | ||
|
|
af854ece5f | ||
|
|
803f864424 | ||
|
|
f672f1a740 | ||
|
|
f20422c53d | ||
|
|
068e686f0c | ||
|
|
3798eb8efd | ||
|
|
d27e4ed963 | ||
|
|
de7c287c84 | ||
|
|
550a98b34c | ||
|
|
ffb32ae363 |
No files matched your search
@@ -24,8 +24,8 @@ inputs:
|
||||
description: "Github token"
|
||||
required: true
|
||||
use_cache:
|
||||
description: "Whether to setup github actions cache (not implemented currently)"
|
||||
default: false
|
||||
description: "Whether to setup magic-nix-cache"
|
||||
default: true
|
||||
required: false
|
||||
runs:
|
||||
using: "composite"
|
||||
@@ -122,3 +122,10 @@ runs:
|
||||
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
|
||||
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
|
||||
extra-conf: ${{ inputs.extra_nix_config }}
|
||||
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
|
||||
if: ${{ inputs.use_cache == 'true' }}
|
||||
with:
|
||||
diagnostic-endpoint: ''
|
||||
use-flakehub: false
|
||||
use-gha-cache: true
|
||||
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
|
||||
@@ -1,8 +0,0 @@
|
||||
---
|
||||
synopsis: "C API: New store API methods"
|
||||
prs: [14766]
|
||||
---
|
||||
|
||||
The C API now includes additional methods:
|
||||
|
||||
- `nix_store_query_path_from_hash_part()` - Get the full store path given its hash part
|
||||
@@ -1,10 +0,0 @@
|
||||
---
|
||||
synopsis: "New setting `ignore-gc-delete-failure` for local stores"
|
||||
prs: [15054]
|
||||
---
|
||||
|
||||
A new local store setting [`ignore-gc-delete-failure`](@docroot@/store/types/local-store.md#store-local-store-ignore-gc-delete-failure) has been added.
|
||||
When enabled, garbage collection will log warnings instead of failing when it cannot delete store paths.
|
||||
This is useful when running Nix as an unprivileged user that may not have write access to all paths in the store.
|
||||
|
||||
This setting is experimental and requires the [`local-overlay-store`](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store) experimental feature.
|
||||
@@ -1,15 +0,0 @@
|
||||
---
|
||||
synopsis: Support HTTPS binary caches using mTLS (client certificate) authentication
|
||||
issues: [13002]
|
||||
prs: [13030]
|
||||
---
|
||||
|
||||
Added support for `tls-certificate` and `tls-private-key` options in substituter URLs.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
https://substituter.invalid?tls-certificate=/path/to/cert.pem&tls-private-key=/path/to/key.pem
|
||||
```
|
||||
|
||||
When these options are configured, Nix will use this certificate/private key pair to authenticate to the server.
|
||||
@@ -135,9 +135,7 @@
|
||||
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
|
||||
- [Store Path Specification](protocols/store-path.md)
|
||||
- [Nix Archive (NAR) Format](protocols/nix-archive/index.md)
|
||||
- [Nix Cache Info Format](protocols/nix-cache-info.md)
|
||||
- [Derivation "ATerm" file format](protocols/derivation-aterm.md)
|
||||
- [Nix32 Encoding](protocols/nix32.md)
|
||||
- [C API](c-api.md)
|
||||
- [Glossary](glossary.md)
|
||||
- [Development](development/index.md)
|
||||
|
||||
@@ -57,6 +57,11 @@ Most Nix commands interpret the following environment variables:
|
||||
|
||||
Overrides the location of the Nix store (default `prefix/store`).
|
||||
|
||||
- <span id="env-NIX_DATA_DIR">[`NIX_DATA_DIR`](#env-NIX_DATA_DIR)</span>
|
||||
|
||||
Overrides the location of the Nix static data directory (default
|
||||
`prefix/share`).
|
||||
|
||||
- <span id="env-NIX_LOG_DIR">[`NIX_LOG_DIR`](#env-NIX_LOG_DIR)</span>
|
||||
|
||||
Overrides the location of the Nix log directory (default
|
||||
|
||||
@@ -6,23 +6,14 @@ It is broken up into multiple Meson packages, which are optionally combined in a
|
||||
There are no mandatory extra steps to the building process:
|
||||
generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/NixOS/nix.git
|
||||
cd nix
|
||||
meson setup build
|
||||
cd build
|
||||
ninja
|
||||
(sudo) ninja install
|
||||
```
|
||||
|
||||
The installation path can be specified by passing `-Dprefix=prefix`
|
||||
to `meson setup build`. The default installation directory is `/usr/local`. You
|
||||
The installation path can be specified by passing the `-Dprefix=prefix`
|
||||
to `configure`. The default installation directory is `/usr/local`. You
|
||||
can change this to any location you like. You must have write permission
|
||||
to the *prefix* path.
|
||||
|
||||
Nix keeps its *store* (the place where packages are stored) in
|
||||
`/nix/store` by default. This can be changed using
|
||||
`-Dlibstore:store-dir=path`.
|
||||
`-Dstore-dir=path`.
|
||||
|
||||
> **Warning**
|
||||
>
|
||||
|
||||
@@ -338,7 +338,7 @@ Here is more information on the `output*` attributes, and what values they may b
|
||||
This will specify the output hash of the single output of a [fixed-output derivation].
|
||||
|
||||
The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/).
|
||||
The ["nix32" encoding](@docroot@/protocols/nix32.md) is Nix's variant of base-32 encoding.
|
||||
The "nix32" encoding is an adaptation of base-32 encoding.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
|
||||
@@ -19,16 +19,17 @@ whatever port you like:
|
||||
$ nix-serve -p 8080
|
||||
```
|
||||
|
||||
To check whether it works, try fetching the [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) file on the client:
|
||||
To check whether it works, try the following on the client:
|
||||
|
||||
```console
|
||||
$ curl http://avalon:8080/nix-cache-info
|
||||
StoreDir: /nix/store
|
||||
WantMassQuery: 1
|
||||
Priority: 30
|
||||
```
|
||||
|
||||
When writing to a binary cache (e.g., with [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md)), Nix creates [`nix-cache-info`](@docroot@/protocols/nix-cache-info.md) automatically if it doesn't exist.
|
||||
which should print something like:
|
||||
|
||||
StoreDir: /nix/store
|
||||
WantMassQuery: 1
|
||||
Priority: 30
|
||||
|
||||
On the client side, you can tell Nix to use your binary cache using
|
||||
`--substituters`, e.g.:
|
||||
|
||||
@@ -24,7 +24,7 @@ description: |
|
||||
|
||||
The format follows this pattern: `${digest}-${name}`
|
||||
|
||||
- **hash**: Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
|
||||
- **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
|
||||
- **name**: The package name and optional version/suffix information
|
||||
|
||||
type: string
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
# Nix Cache Info Format
|
||||
|
||||
The `nix-cache-info` file is a metadata file at the root of a [binary cache](@docroot@/package-management/binary-cache-substituter.md) (e.g., `https://cache.example.com/nix-cache-info`).
|
||||
|
||||
MIME type: `text/x-nix-cache-info`
|
||||
|
||||
## Format
|
||||
|
||||
Line-based key-value format:
|
||||
|
||||
```
|
||||
Key: value
|
||||
```
|
||||
|
||||
Leading and trailing whitespace is trimmed from values.
|
||||
Lines without a colon are ignored.
|
||||
Unknown keys are silently ignored.
|
||||
|
||||
## Fields
|
||||
|
||||
### `StoreDir`
|
||||
|
||||
The Nix store directory path that this cache was built for (e.g., `/nix/store`).
|
||||
|
||||
If present, Nix verifies that this matches the client's store directory:
|
||||
|
||||
```
|
||||
error: binary cache 'https://example.com' is for Nix stores with prefix '/nix/store', not '/home/user/nix/store'
|
||||
```
|
||||
|
||||
### `WantMassQuery`
|
||||
|
||||
`1` or `0`. Sets the default for [`want-mass-query`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-want-mass-query).
|
||||
|
||||
### `Priority`
|
||||
|
||||
Integer. Sets the default for [`priority`](@docroot@/store/types/http-binary-cache-store.md#store-http-binary-cache-store-priority).
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
StoreDir: /nix/store
|
||||
WantMassQuery: 1
|
||||
Priority: 30
|
||||
```
|
||||
|
||||
## Caching Behavior
|
||||
|
||||
Nix caches `nix-cache-info` in the [cache directory](@docroot@/command-ref/env-common.md#env-NIX_CACHE_HOME) with a 7-day TTL.
|
||||
|
||||
## See Also
|
||||
|
||||
- [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md)
|
||||
- [Serving a Nix store via HTTP](@docroot@/package-management/binary-cache-substituter.md)
|
||||
- [`substituters`](@docroot@/command-ref/conf-file.md#conf-substituters)
|
||||
@@ -1,19 +0,0 @@
|
||||
# Nix32 Encoding
|
||||
|
||||
Nix32 is Nix's variant of base-32 encoding, used for [store path digests](@docroot@/protocols/store-path.md), hash output via [`nix hash`](@docroot@/command-ref/new-cli/nix3-hash.md), and the [`outputHash`](@docroot@/language/advanced-attributes.md#adv-attr-outputHash) derivation attribute.
|
||||
|
||||
## Alphabet
|
||||
|
||||
The Nix32 alphabet consists of these 32 characters:
|
||||
|
||||
```
|
||||
0 1 2 3 4 5 6 7 8 9 a b c d f g h i j k l m n p q r s v w x y z
|
||||
```
|
||||
|
||||
The letters `e`, `o`, `u`, and `t` are omitted.
|
||||
|
||||
## Byte Order
|
||||
|
||||
Nix32 encoding processes the hash bytes from the end (last byte first), while base-16 encoding processes from the beginning (first byte first).
|
||||
|
||||
Consequently, the string sort order is determined primarily by the first bytes for base-16, and by the last bytes for Nix32.
|
||||
@@ -20,11 +20,12 @@ where
|
||||
|
||||
- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory)
|
||||
|
||||
- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`.
|
||||
- `digest` = base-32 representation of the compressed to 160 bits [SHA-256] hash of `fingerprint`
|
||||
|
||||
Nix uses a custom base-32 encoding called [Nix32](@docroot@/protocols/nix32.md).
|
||||
|
||||
For the definition of the hash compression algorithm, please refer to section 5.1 of the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf).
|
||||
For the definition of the hash compression algorithm, please refer to the section 5.1 of
|
||||
the [Nix thesis](https://edolstra.github.io/pubs/phd-thesis.pdf), which also defines the
|
||||
specifics of base-32 encoding. Note that base-32 encoding processes the hash bytestring from
|
||||
the end, while base-16 processes in from the beginning.
|
||||
|
||||
## Fingerprint
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ A store path is rendered to a file system path as the concatenation of
|
||||
|
||||
- [Store directory](#store-directory) (typically `/nix/store`)
|
||||
- Path separator (`/`)
|
||||
- Digest rendered in [Nix32](@docroot@/protocols/nix32.md), a variant of base-32 (20 hash bytes become 32 ASCII characters)
|
||||
- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
|
||||
- Hyphen (`-`)
|
||||
- Name
|
||||
|
||||
|
||||
Generated
+5
-5
@@ -3,15 +3,15 @@
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1767039857,
|
||||
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||
"owner": "NixOS",
|
||||
"lastModified": 1733328505,
|
||||
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
|
||||
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
|
||||
inputs.flake-compat = {
|
||||
url = "github:NixOS/flake-compat";
|
||||
url = "github:edolstra/flake-compat";
|
||||
flake = false;
|
||||
};
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
|
||||
KillMode=process
|
||||
LimitNOFILE=1048576
|
||||
TasksMax=1048576
|
||||
Delegate=
|
||||
Delegate=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -131,7 +131,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
|
||||
|
||||
availableComponents = lib.filterAttrs (
|
||||
k: v: lib.meta.availableOn pkgs.stdenv.hostPlatform v
|
||||
k: v: lib.meta.availableOn pkgs.hostPlatform v
|
||||
) allComponents;
|
||||
|
||||
activeComponents = buildInputsClosureCond isInternal (
|
||||
@@ -300,7 +300,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
lib.filter (x: !isInternal x) (
|
||||
lib.lists.concatMap (
|
||||
# Nix manual has a build-time dependency on nix, but we
|
||||
# don't want to do a native build just to enter the cross
|
||||
# don't want to do a native build just to enter the ross
|
||||
# dev shell.
|
||||
#
|
||||
# TODO: think of a more principled fix for this.
|
||||
@@ -323,7 +323,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
pkgs.buildPackages.shellcheck
|
||||
pkgs.buildPackages.include-what-you-use
|
||||
]
|
||||
++ lib.optional stdenv.hostPlatform.isUnix pkgs.buildPackages.gdb
|
||||
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
|
||||
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
|
||||
lib.hiPrio pkgs.buildPackages.clang-tools
|
||||
)
|
||||
@@ -341,7 +341,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
|
||||
buildInputs =
|
||||
# TODO change Nixpkgs to mark gbenchmark as building on Windows
|
||||
lib.optional stdenv.hostPlatform.isUnix pkgs.gbenchmark
|
||||
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
|
||||
++ dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
||||
)
|
||||
|
||||
@@ -300,8 +300,6 @@ MixDefaultProfile::MixDefaultProfile()
|
||||
profile = getDefaultProfile().string();
|
||||
}
|
||||
|
||||
static constexpr auto environmentVariablesCategory = "Options that change environment variables";
|
||||
|
||||
MixEnvironment::MixEnvironment()
|
||||
: ignoreEnvironment(false)
|
||||
{
|
||||
|
||||
@@ -157,7 +157,7 @@ MixFlakeOptions::MixFlakeOptions()
|
||||
.category = category,
|
||||
.labels = {"flake-lock-path"},
|
||||
.handler = {[&](std::string lockFilePath) {
|
||||
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
|
||||
lockFlags.referenceLockFilePath = {makeFSSourceAccessor(absPath(lockFilePath)), CanonPath::root};
|
||||
}},
|
||||
.completer = completePath,
|
||||
});
|
||||
|
||||
+1
-1
@@ -795,7 +795,7 @@ void NixRepl::loadFiles()
|
||||
loadedFiles.clear();
|
||||
|
||||
for (auto & i : old) {
|
||||
notice("Loading %1%...", PathFmt(i));
|
||||
notice("Loading '%1%'...", i);
|
||||
loadFile(i);
|
||||
}
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ struct AttrDb
|
||||
|
||||
auto dbPath = cacheDir / (fingerprint.to_string(HashFormat::Base16, false) + ".sqlite");
|
||||
|
||||
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
|
||||
state->db = SQLite(dbPath);
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
|
||||
@@ -139,7 +139,7 @@ public:
|
||||
, profileFd([&]() {
|
||||
AutoCloseFD fd = toDescriptor(open(profileFile.string().c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0660));
|
||||
if (!fd)
|
||||
throw SysError("opening file %s", PathFmt(profileFile));
|
||||
throw SysError("opening file %s", profileFile);
|
||||
return fd;
|
||||
}())
|
||||
, posCache(state)
|
||||
|
||||
@@ -1002,12 +1002,6 @@ public:
|
||||
[[nodiscard]] StringMap
|
||||
realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
|
||||
|
||||
/**
|
||||
* Coerce `v` to a path and realise it, i.e. build anything in the value's string context using `realiseContext()`.
|
||||
*/
|
||||
SourcePath realisePath(
|
||||
const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full);
|
||||
|
||||
/**
|
||||
* Realise the given string with context, and return the string with outputs instead of downstream output
|
||||
* placeholders.
|
||||
|
||||
+16
-12
@@ -157,20 +157,24 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
|
||||
return res;
|
||||
}
|
||||
|
||||
SourcePath EvalState::realisePath(const PosIdx pos, Value & v, std::optional<SymlinkResolution> resolveSymlinks)
|
||||
static SourcePath realisePath(
|
||||
EvalState & state,
|
||||
const PosIdx pos,
|
||||
Value & v,
|
||||
std::optional<SymlinkResolution> resolveSymlinks = SymlinkResolution::Full)
|
||||
{
|
||||
NixStringContext context;
|
||||
|
||||
auto path = coerceToPath(noPos, v, context, "while realising the context of a path");
|
||||
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
|
||||
|
||||
try {
|
||||
if (!context.empty() && path.accessor == rootFS) {
|
||||
auto rewrites = realiseContext(context);
|
||||
if (!context.empty() && path.accessor == state.rootFS) {
|
||||
auto rewrites = state.realiseContext(context);
|
||||
path = {path.accessor, CanonPath(rewriteStrings(path.path.abs(), rewrites))};
|
||||
}
|
||||
return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path;
|
||||
} catch (Error & e) {
|
||||
e.addTrace(positions[pos], "while realising the context of path '%s'", path);
|
||||
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -290,7 +294,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path,
|
||||
argument. */
|
||||
static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v)
|
||||
{
|
||||
auto path = state.realisePath(pos, vPath, std::nullopt);
|
||||
auto path = realisePath(state, pos, vPath, std::nullopt);
|
||||
auto path2 = path.path.abs();
|
||||
|
||||
// FIXME
|
||||
@@ -443,7 +447,7 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v);
|
||||
/* Load a ValueInitializer from a DSO and return whatever it initializes */
|
||||
void prim_importNative(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = state.realisePath(pos, *args[0]);
|
||||
auto path = realisePath(state, pos, *args[0]);
|
||||
|
||||
std::string sym(
|
||||
state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative"));
|
||||
@@ -1967,7 +1971,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value ** args,
|
||||
arg.type() == nString && (arg.string_view().ends_with("/") || arg.string_view().ends_with("/."));
|
||||
|
||||
auto symlinkResolution = mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors;
|
||||
auto path = state.realisePath(pos, arg, symlinkResolution);
|
||||
auto path = realisePath(state, pos, arg, symlinkResolution);
|
||||
|
||||
auto st = path.maybeLstat();
|
||||
auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory);
|
||||
@@ -2074,7 +2078,7 @@ static RegisterPrimOp primop_dirOf({
|
||||
/* Return the contents of a file as a string. */
|
||||
static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = state.realisePath(pos, *args[0]);
|
||||
auto path = realisePath(state, pos, *args[0]);
|
||||
auto s = path.readFile();
|
||||
if (s.find((char) 0) != std::string::npos)
|
||||
state.error<EvalError>("the contents of the file '%1%' cannot be represented as a Nix string", path)
|
||||
@@ -2309,7 +2313,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
|
||||
if (!ha)
|
||||
state.error<EvalError>("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow();
|
||||
|
||||
auto path = state.realisePath(pos, *args[1]);
|
||||
auto path = realisePath(state, pos, *args[1]);
|
||||
|
||||
v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
|
||||
}
|
||||
@@ -2361,7 +2365,7 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
|
||||
|
||||
static void prim_readFileType(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = state.realisePath(pos, *args[0], std::nullopt);
|
||||
auto path = realisePath(state, pos, *args[0], std::nullopt);
|
||||
/* Retrieve the directory entry type and stringize it. */
|
||||
v = fileTypeToString(state, path.lstat().type);
|
||||
}
|
||||
@@ -2379,7 +2383,7 @@ static RegisterPrimOp primop_readFileType({
|
||||
/* Read a directory (without . or ..) */
|
||||
static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Value & v)
|
||||
{
|
||||
auto path = state.realisePath(pos, *args[0]);
|
||||
auto path = realisePath(state, pos, *args[0]);
|
||||
|
||||
// Retrieve directory entries for all nodes in a directory.
|
||||
// This is similar to `getFileType` but is optimized to reduce system calls
|
||||
|
||||
@@ -41,7 +41,7 @@ struct CacheImpl : Cache
|
||||
auto dbPath = (getCacheDir() / "fetcher-cache-v4.sqlite").string();
|
||||
createDirs(dirOf(dbPath));
|
||||
|
||||
state->db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
|
||||
state->db = SQLite(dbPath);
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
|
||||
@@ -489,11 +489,11 @@ void InputScheme::clone(
|
||||
const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
|
||||
{
|
||||
if (std::filesystem::exists(destDir))
|
||||
throw Error("cannot clone into existing path %s", PathFmt(destDir));
|
||||
throw Error("cannot clone into existing path %s", destDir);
|
||||
|
||||
auto [accessor, input2] = getAccessor(settings, store, input);
|
||||
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), PathFmt(destDir)));
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
|
||||
|
||||
RestoreSink sink(/*startFsync=*/false);
|
||||
sink.dstPath = destDir;
|
||||
|
||||
@@ -10,6 +10,12 @@ std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(co
|
||||
return next->getPhysicalPath(prefix / path);
|
||||
}
|
||||
|
||||
std::string FilteringSourceAccessor::readFile(const CanonPath & path)
|
||||
{
|
||||
checkAccess(path);
|
||||
return next->readFile(prefix / path);
|
||||
}
|
||||
|
||||
void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||
{
|
||||
checkAccess(path);
|
||||
|
||||
@@ -273,7 +273,7 @@ void Fetch::fetch(
|
||||
+ "/" + pointer->oid;
|
||||
std::filesystem::path cachePath = cacheDir / key;
|
||||
if (pathExists(cachePath)) {
|
||||
debug("using cache entry %s -> %s", key, PathFmt(cachePath));
|
||||
debug("using cache entry %s -> %s", key, cachePath);
|
||||
sink(readFile(cachePath));
|
||||
return;
|
||||
}
|
||||
@@ -301,7 +301,7 @@ void Fetch::fetch(
|
||||
sizeCallback(size);
|
||||
downloadToSink(ourl, authHeader, sink, sha256, size);
|
||||
|
||||
debug("creating cache entry %s -> %s", key, PathFmt(cachePath));
|
||||
debug("creating cache entry %s -> %s", key, cachePath);
|
||||
if (!pathExists(cachePath.parent_path()))
|
||||
createDirs(cachePath.parent_path());
|
||||
writeFile(cachePath, sink.s);
|
||||
|
||||
@@ -209,14 +209,14 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
|
||||
return;
|
||||
|
||||
if (!options.create)
|
||||
throw Error("Git repository %s does not exist.", PathFmt(path));
|
||||
throw Error("Git repository %s does not exist.", path);
|
||||
|
||||
std::filesystem::path tmpDir = createTempDir(path.parent_path());
|
||||
AutoDelete delTmpDir(tmpDir, true);
|
||||
Repository tmpRepo;
|
||||
|
||||
if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
|
||||
throw Error("creating Git repository %s: %s", PathFmt(path), git_error_last()->message);
|
||||
throw Error("creating Git repository %s: %s", path, git_error_last()->message);
|
||||
try {
|
||||
std::filesystem::rename(tmpDir, path);
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
@@ -226,7 +226,7 @@ static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options op
|
||||
|| e.code() == std::errc::directory_not_empty) {
|
||||
return;
|
||||
} else
|
||||
throw SysError("moving temporary git repository from %s to %s", PathFmt(tmpDir), PathFmt(path));
|
||||
throw SysError("moving temporary git repository from %s to %s", tmpDir, path);
|
||||
}
|
||||
// we successfully moved the repository, so the temporary directory no longer exists.
|
||||
delTmpDir.cancel();
|
||||
@@ -266,7 +266,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||
|
||||
initRepoAtomically(path, options);
|
||||
if (git_repository_open(Setter(repo), path.string().c_str()))
|
||||
throw Error("opening Git repository %s: %s", PathFmt(path), git_error_last()->message);
|
||||
throw Error("opening Git repository %s: %s", path, git_error_last()->message);
|
||||
|
||||
ObjectDb odb;
|
||||
if (options.packfilesOnly) {
|
||||
@@ -766,7 +766,7 @@ struct GitSourceAccessor : SourceAccessor
|
||||
{
|
||||
}
|
||||
|
||||
void readBlob(const CanonPath & path, bool symlink, Sink & sink, std::function<void(uint64_t)> sizeCallback)
|
||||
std::string readBlob(const CanonPath & path, bool symlink)
|
||||
{
|
||||
auto state(state_.lock());
|
||||
|
||||
@@ -785,22 +785,16 @@ struct GitSourceAccessor : SourceAccessor
|
||||
e.addTrace({}, "while smudging git-lfs file '%s'", path);
|
||||
throw;
|
||||
}
|
||||
sizeCallback(s.s.size());
|
||||
StringSource source{s.s};
|
||||
source.drainInto(sink);
|
||||
return;
|
||||
return s.s;
|
||||
}
|
||||
}
|
||||
|
||||
auto view = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
|
||||
sizeCallback(view.size());
|
||||
StringSource source{view};
|
||||
source.drainInto(sink);
|
||||
return std::string((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
|
||||
}
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
|
||||
std::string readFile(const CanonPath & path) override
|
||||
{
|
||||
return readBlob(path, false, sink, sizeCallback);
|
||||
return readBlob(path, false);
|
||||
}
|
||||
|
||||
bool pathExists(const CanonPath & path) override
|
||||
@@ -867,9 +861,7 @@ struct GitSourceAccessor : SourceAccessor
|
||||
|
||||
std::string readLink(const CanonPath & path) override
|
||||
{
|
||||
StringSink s;
|
||||
readBlob(path, true, s, [&](uint64_t size) { s.s.reserve(size); });
|
||||
return std::move(s.s);
|
||||
return readBlob(path, true);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1433,7 +1425,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
|
||||
auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
|
||||
try {
|
||||
writeFull(fdTemp.get(), configS);
|
||||
} catch (SystemError & e) {
|
||||
} catch (SysError & e) {
|
||||
e.addTrace({}, "while writing .gitmodules file to temporary file");
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -72,10 +72,10 @@ std::optional<std::string> readHead(const std::filesystem::path & path)
|
||||
if (const auto parseResult = git::parseLsRemoteLine(line); parseResult && parseResult->reference == "HEAD") {
|
||||
switch (parseResult->kind) {
|
||||
case git::LsRemoteRefLine::Kind::Symbolic:
|
||||
debug("resolved HEAD ref '%s' for repo %s", parseResult->target, PathFmt(path));
|
||||
debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path);
|
||||
break;
|
||||
case git::LsRemoteRefLine::Kind::Object:
|
||||
debug("resolved HEAD rev '%s' for repo %s", parseResult->target, PathFmt(path));
|
||||
debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path);
|
||||
break;
|
||||
}
|
||||
return parseResult->target;
|
||||
@@ -753,10 +753,9 @@ struct GitInputScheme : InputScheme
|
||||
"\n"
|
||||
"git -C %2% add \"%1%\"",
|
||||
path.rel(),
|
||||
PathFmt(repoPath));
|
||||
repoPath);
|
||||
else
|
||||
return RestrictedPathError(
|
||||
"Path '%s' does not exist in Git repository %s.", path.rel(), PathFmt(repoPath));
|
||||
return RestrictedPathError("Path '%s' does not exist in Git repository %s.", path.rel(), repoPath);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -849,7 +848,7 @@ struct GitInputScheme : InputScheme
|
||||
if (!input.getRev())
|
||||
setWriteTime(localRefFile, now, now);
|
||||
} catch (Error & e) {
|
||||
warn("could not update mtime for file %s: %s", PathFmt(localRefFile), e.info().msg);
|
||||
warn("could not update mtime for file %s: %s", localRefFile, e.info().msg);
|
||||
}
|
||||
if (!originalRef && !storeCachedHead(repoUrl.to_string(), shallow, ref))
|
||||
warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
|
||||
|
||||
@@ -34,7 +34,7 @@ struct FilteringSourceAccessor : SourceAccessor
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
|
||||
|
||||
using SourceAccessor::readFile;
|
||||
std::string readFile(const CanonPath & path) override;
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
|
||||
|
||||
|
||||
@@ -213,26 +213,24 @@ struct MercurialInputScheme : InputScheme
|
||||
runHg({"status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0"}),
|
||||
"\0"s);
|
||||
|
||||
std::filesystem::path actualPath(absPath(actualUrl));
|
||||
auto accessor = makeFSSourceAccessor(absPath(actualUrl));
|
||||
|
||||
PathFilter filter = [&](const Path & p) -> bool {
|
||||
assert(hasPrefix(p, actualPath.string()));
|
||||
std::string file(p, actualPath.string().size() + 1);
|
||||
auto cp = CanonPath(p);
|
||||
auto st = accessor->lstat(cp);
|
||||
|
||||
auto st = lstat(p);
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
auto prefix = file + "/";
|
||||
if (st.type == SourceAccessor::tDirectory) {
|
||||
auto prefix = cp.rel() + "/";
|
||||
auto i = files.lower_bound(prefix);
|
||||
return i != files.end() && hasPrefix(*i, prefix);
|
||||
}
|
||||
|
||||
return files.count(file);
|
||||
return files.count(cp.rel());
|
||||
};
|
||||
|
||||
auto storePath = store.addToStore(
|
||||
input.getName(),
|
||||
{getFSSourceAccessor(), CanonPath(actualPath.string())},
|
||||
{accessor, CanonPath::root},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256,
|
||||
{},
|
||||
@@ -334,7 +332,7 @@ struct MercurialInputScheme : InputScheme
|
||||
|
||||
deletePath(tmpDir / ".hg_archival.txt");
|
||||
|
||||
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())});
|
||||
auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root});
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"revCount", (uint64_t) revCount},
|
||||
|
||||
@@ -154,7 +154,7 @@ struct PathInputScheme : InputScheme
|
||||
|
||||
time_t mtime = 0;
|
||||
if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", PathFmt(absPath)));
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
|
||||
// FIXME: try to substitute storePath.
|
||||
auto src = sinkToSource(
|
||||
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
|
||||
|
||||
@@ -114,22 +114,15 @@ static DownloadTarballResult downloadTarball_(
|
||||
// it is not in fact a tarball.
|
||||
if (url.scheme == "file") {
|
||||
std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path);
|
||||
if (!localPath.is_absolute()) {
|
||||
throw Error(
|
||||
"tarball '%s' must use an absolute path. "
|
||||
"The 'file' scheme does not support relative paths.",
|
||||
url);
|
||||
}
|
||||
if (!exists(localPath)) {
|
||||
throw Error("tarball %s does not exist.", PathFmt(localPath));
|
||||
throw Error("tarball '%s' does not exist.", localPath);
|
||||
}
|
||||
if (is_directory(localPath)) {
|
||||
if (exists(localPath / ".git")) {
|
||||
throw Error(
|
||||
"tarball %s is a git repository, not a tarball. Please use `git+file` as the scheme.",
|
||||
PathFmt(localPath));
|
||||
"tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath);
|
||||
}
|
||||
throw Error("tarball %s is a directory, not a file.", PathFmt(localPath));
|
||||
throw Error("tarball '%s' is a directory, not a file.", localPath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -377,7 +377,6 @@ TEST_F(nix_api_store_test, nix_api_load_flake_with_flags)
|
||||
assert_ctx_ok();
|
||||
ASSERT_EQ("Claire", helloStr);
|
||||
|
||||
nix_locked_flake_free(lockedFlake);
|
||||
nix_flake_reference_parse_flags_free(parseFlags);
|
||||
nix_flake_lock_flags_free(lockFlags);
|
||||
nix_flake_reference_free(flakeReference);
|
||||
|
||||
@@ -850,11 +850,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
|
||||
auto s = chomp(diff);
|
||||
if (lockFileExists) {
|
||||
if (s.empty())
|
||||
warn("updating lock file %s", PathFmt(outputLockFilePath));
|
||||
warn("updating lock file %s", outputLockFilePath);
|
||||
else
|
||||
warn("updating lock file %s:\n%s", PathFmt(outputLockFilePath), s);
|
||||
warn("updating lock file %s:\n%s", outputLockFilePath, s);
|
||||
} else
|
||||
warn("creating lock file %s: \n%s", PathFmt(outputLockFilePath), s);
|
||||
warn("creating lock file %s: \n%s", outputLockFilePath, s);
|
||||
|
||||
std::optional<std::string> commitMessage = std::nullopt;
|
||||
|
||||
|
||||
@@ -83,8 +83,8 @@ void initPlugins()
|
||||
checkInterrupt();
|
||||
pluginFiles.emplace_back(ent.path());
|
||||
}
|
||||
} catch (SystemError & e) {
|
||||
if (!e.is(std::errc::not_a_directory))
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo != ENOTDIR)
|
||||
throw;
|
||||
pluginFiles.emplace_back(pluginFile);
|
||||
}
|
||||
@@ -95,7 +95,7 @@ void initPlugins()
|
||||
#ifndef _WIN32 // TODO implement via DLL loading on Windows
|
||||
void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
|
||||
if (!handle)
|
||||
throw Error("could not dynamically open plugin file %s: %s", PathFmt(file), dlerror());
|
||||
throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
|
||||
|
||||
/* Older plugins use a statically initialized object to run their code.
|
||||
Newer plugins can also export nix_plugin_entry() */
|
||||
@@ -103,7 +103,7 @@ void initPlugins()
|
||||
if (nix_plugin_entry)
|
||||
nix_plugin_entry();
|
||||
#else
|
||||
throw Error("could not dynamically open plugin file %s", PathFmt(file));
|
||||
throw Error("could not dynamically open plugin file '%s'", file);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,10 +304,11 @@ void printVersion(const std::string & programName)
|
||||
std::cout << "System type: " << settings.thisSystem << "\n";
|
||||
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
|
||||
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
|
||||
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf").string() << "\n";
|
||||
std::cout << "System configuration file: " << (settings.nixConfDir / "nix.conf") << "\n";
|
||||
std::cout << "User configuration files: " << concatStringsSep(":", settings.nixUserConfFiles) << "\n";
|
||||
std::cout << "Store directory: " << settings.nixStore << "\n";
|
||||
std::cout << "State directory: " << settings.nixStateDir << "\n";
|
||||
std::cout << "Data directory: " << settings.nixDataDir << "\n";
|
||||
}
|
||||
throw Exit();
|
||||
}
|
||||
|
||||
@@ -338,20 +338,4 @@ nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store *
|
||||
NIXC_CATCH_ERRS_NULL
|
||||
}
|
||||
|
||||
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash)
|
||||
{
|
||||
if (context)
|
||||
context->last_err_code = NIX_OK;
|
||||
try {
|
||||
std::optional<nix::StorePath> s = store->ptr->queryPathFromHashPart(hash);
|
||||
|
||||
if (!s.has_value()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return new StorePath{std::move(s.value())};
|
||||
}
|
||||
NIXC_CATCH_ERRS_NULL
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
@@ -259,17 +259,6 @@ nix_err nix_store_get_fs_closure(
|
||||
*/
|
||||
nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path);
|
||||
|
||||
/**
|
||||
* @brief Query the full store path given the hash part of a valid store
|
||||
* path, or empty if no matching path is found.
|
||||
*
|
||||
* @param[out] context Optional, stores error information
|
||||
* @param[in] store nix store reference
|
||||
* @param[in] hash Hash part of path as a string
|
||||
* @return Store path reference, NULL if no matching path is found.
|
||||
*/
|
||||
StorePath * nix_store_query_path_from_hash_part(nix_c_context * context, Store * store, const char * hash);
|
||||
|
||||
// cffi end
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
#include "nix/store/tests/https-store.hh"
|
||||
|
||||
#include <thread>
|
||||
|
||||
namespace nix::testing {
|
||||
|
||||
void TestHttpBinaryCacheStore::init()
|
||||
{
|
||||
BinaryCacheStore::init();
|
||||
}
|
||||
|
||||
ref<TestHttpBinaryCacheStore> TestHttpBinaryCacheStoreConfig::openTestStore() const
|
||||
{
|
||||
auto store = make_ref<TestHttpBinaryCacheStore>(
|
||||
ref{// FIXME we shouldn't actually need a mutable config
|
||||
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())});
|
||||
store->init();
|
||||
return store;
|
||||
}
|
||||
|
||||
void HttpsBinaryCacheStoreTest::openssl(Strings args)
|
||||
{
|
||||
runProgram("openssl", /*lookupPath=*/true, args);
|
||||
}
|
||||
|
||||
void HttpsBinaryCacheStoreTest::SetUp()
|
||||
{
|
||||
LibStoreNetworkTest::SetUp();
|
||||
|
||||
#ifdef _WIN32
|
||||
GTEST_SKIP() << "HTTPS store tests are not supported on Windows";
|
||||
#endif
|
||||
|
||||
tmpDir = createTempDir();
|
||||
cacheDir = tmpDir / "cache";
|
||||
delTmpDir = std::make_unique<AutoDelete>(tmpDir);
|
||||
|
||||
localCacheStore =
|
||||
make_ref<LocalBinaryCacheStoreConfig>("file", cacheDir.string(), LocalBinaryCacheStoreConfig::Params{})
|
||||
->openStore();
|
||||
|
||||
caCert = tmpDir / "ca.crt";
|
||||
caKey = tmpDir / "ca.key";
|
||||
serverCert = tmpDir / "server.crt";
|
||||
serverKey = tmpDir / "server.key";
|
||||
clientCert = tmpDir / "client.crt";
|
||||
clientKey = tmpDir / "client.key";
|
||||
|
||||
// clang-format off
|
||||
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", caKey.string()});
|
||||
openssl({"req", "-new", "-x509", "-days", "1", "-key", caKey.string(), "-out", caCert.string(), "-subj", "/CN=TestCA"});
|
||||
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", serverKey.string()});
|
||||
openssl({"req", "-new", "-key", serverKey.string(), "-out", (tmpDir / "server.csr").string(), "-subj", "/CN=localhost"});
|
||||
openssl({"x509", "-req", "-in", (tmpDir / "server.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", serverCert.string(), "-days", "1"});
|
||||
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", clientKey.string()});
|
||||
openssl({"req", "-new", "-key", clientKey.string(), "-out", (tmpDir / "client.csr").string(), "-subj", "/CN=TestClient"});
|
||||
openssl({"x509", "-req", "-in", (tmpDir / "client.csr").string(), "-CA", caCert.string(), "-CAkey", caKey.string(), "-CAcreateserial", "-out", clientCert.string(), "-days", "1"});
|
||||
// clang-format on
|
||||
|
||||
#ifndef _WIN32 /* FIXME: Can't yet start processes on windows */
|
||||
auto args = serverArgs();
|
||||
serverPid = startProcess(
|
||||
[&] {
|
||||
if (chdir(cacheDir.c_str()) == -1)
|
||||
_exit(1);
|
||||
std::vector<char *> argv;
|
||||
argv.push_back(const_cast<char *>("openssl"));
|
||||
for (auto & a : args)
|
||||
argv.push_back(const_cast<char *>(a.c_str()));
|
||||
argv.push_back(nullptr);
|
||||
execvp("openssl", argv.data());
|
||||
_exit(1);
|
||||
},
|
||||
{.dieWithParent = true});
|
||||
#endif
|
||||
|
||||
/* As an optimization, sleep for a bit to allow the server to come up to avoid retrying when connecting.
|
||||
This won't make the tests fail, but does make them run faster. We don't need to overcomplicate by waiting
|
||||
for the port explicitly - this is enough. */
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
||||
|
||||
/* FIXME: Don't use global settings. Tests are not run concurrently, so this is fine for now. */
|
||||
oldCaCert = settings.caFile;
|
||||
settings.caFile = caCert.string();
|
||||
}
|
||||
|
||||
void HttpsBinaryCacheStoreTest::TearDown()
|
||||
{
|
||||
settings.caFile = oldCaCert;
|
||||
serverPid.kill();
|
||||
delTmpDir.reset();
|
||||
}
|
||||
|
||||
std::vector<std::string> HttpsBinaryCacheStoreTest::serverArgs()
|
||||
{
|
||||
return {
|
||||
"s_server",
|
||||
"-accept",
|
||||
std::to_string(port),
|
||||
"-cert",
|
||||
serverCert.string(),
|
||||
"-key",
|
||||
serverKey.string(),
|
||||
"-WWW", /* Serve from current directory. */
|
||||
"-quiet",
|
||||
};
|
||||
}
|
||||
|
||||
std::vector<std::string> HttpsBinaryCacheStoreMtlsTest::serverArgs()
|
||||
{
|
||||
auto args = HttpsBinaryCacheStoreTest::serverArgs();
|
||||
/* With the -Verify option the client must supply a certificate or an error occurs, which is not the
|
||||
case with -verify. */
|
||||
args.insert(args.end(), {"-CAfile", caCert.string(), "-Verify", "1", "-verify_return_error"});
|
||||
return args;
|
||||
}
|
||||
|
||||
ref<TestHttpBinaryCacheStoreConfig> HttpsBinaryCacheStoreTest::makeConfig(BinaryCacheStoreConfig::Params params)
|
||||
{
|
||||
auto res = make_ref<TestHttpBinaryCacheStoreConfig>("https", fmt("localhost:%d", port), std::move(params));
|
||||
res->pathInfoCacheSize = 0; /* We don't want any caching in tests. */
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace nix::testing
|
||||
@@ -1,93 +0,0 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <gmock/gmock.h>
|
||||
|
||||
#include "nix/store/tests/libstore-network.hh"
|
||||
#include "nix/store/http-binary-cache-store.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
#include "nix/store/local-binary-cache-store.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/processes.hh"
|
||||
|
||||
namespace nix::testing {
|
||||
|
||||
class TestHttpBinaryCacheStoreConfig;
|
||||
|
||||
/**
|
||||
* Test shim for testing. We don't want to use the on-disk narinfo cache in unit
|
||||
* tests.
|
||||
*/
|
||||
class TestHttpBinaryCacheStore : public HttpBinaryCacheStore
|
||||
{
|
||||
public:
|
||||
TestHttpBinaryCacheStore(const TestHttpBinaryCacheStore &) = delete;
|
||||
TestHttpBinaryCacheStore(TestHttpBinaryCacheStore &&) = delete;
|
||||
TestHttpBinaryCacheStore & operator=(const TestHttpBinaryCacheStore &) = delete;
|
||||
TestHttpBinaryCacheStore & operator=(TestHttpBinaryCacheStore &&) = delete;
|
||||
|
||||
TestHttpBinaryCacheStore(ref<HttpBinaryCacheStoreConfig> config)
|
||||
: Store{*config}
|
||||
, BinaryCacheStore{*config}
|
||||
, HttpBinaryCacheStore(config)
|
||||
{
|
||||
diskCache = nullptr; /* Disable caching, we'll be creating a new binary cache for each test. */
|
||||
}
|
||||
|
||||
void init() override;
|
||||
};
|
||||
|
||||
class TestHttpBinaryCacheStoreConfig : public HttpBinaryCacheStoreConfig
|
||||
{
|
||||
public:
|
||||
TestHttpBinaryCacheStoreConfig(
|
||||
std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params)
|
||||
: StoreConfig(params)
|
||||
, HttpBinaryCacheStoreConfig(scheme, cacheUri, params)
|
||||
{
|
||||
}
|
||||
|
||||
ref<TestHttpBinaryCacheStore> openTestStore() const;
|
||||
|
||||
ref<Store> openStore() const override
|
||||
{
|
||||
return openTestStore();
|
||||
}
|
||||
};
|
||||
|
||||
class HttpsBinaryCacheStoreTest : public virtual LibStoreNetworkTest
|
||||
{
|
||||
std::unique_ptr<AutoDelete> delTmpDir;
|
||||
|
||||
public:
|
||||
static void SetUpTestSuite()
|
||||
{
|
||||
initLibStore(/*loadConfig=*/false);
|
||||
}
|
||||
|
||||
protected:
|
||||
std::filesystem::path tmpDir, cacheDir;
|
||||
std::filesystem::path caCert, caKey, serverCert, serverKey;
|
||||
std::filesystem::path clientCert, clientKey;
|
||||
std::string oldCaCert;
|
||||
Pid serverPid;
|
||||
uint16_t port = 8443;
|
||||
std::shared_ptr<Store> localCacheStore;
|
||||
|
||||
static void openssl(Strings args);
|
||||
void SetUp() override;
|
||||
void TearDown() override;
|
||||
|
||||
virtual std::vector<std::string> serverArgs();
|
||||
ref<TestHttpBinaryCacheStoreConfig> makeConfig(BinaryCacheStoreConfig::Params params);
|
||||
};
|
||||
|
||||
class HttpsBinaryCacheStoreMtlsTest : public HttpsBinaryCacheStoreTest
|
||||
{
|
||||
protected:
|
||||
std::vector<std::string> serverArgs() override;
|
||||
};
|
||||
|
||||
} // namespace nix::testing
|
||||
@@ -1,39 +0,0 @@
|
||||
#pragma once
|
||||
/// @file
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
namespace nix::testing {
|
||||
|
||||
/**
|
||||
* Whether to run network tests. This is global so that the test harness can
|
||||
* enable this by default if we can run tests in isolation.
|
||||
*/
|
||||
extern bool networkTestsAvailable;
|
||||
|
||||
/**
|
||||
* Set up network tests and, if on linux, create a new network namespace for
|
||||
* tests with a loopback interface. This is to avoid binding to ports in the
|
||||
* host's namespace.
|
||||
*/
|
||||
void setupNetworkTests();
|
||||
|
||||
class LibStoreNetworkTest : public virtual ::testing::Test
|
||||
{
|
||||
protected:
|
||||
void SetUp() override
|
||||
{
|
||||
if (networkTestsAvailable)
|
||||
return;
|
||||
static bool warned = false;
|
||||
if (!warned) {
|
||||
warned = true;
|
||||
GTEST_SKIP()
|
||||
<< "Network tests not enabled by default without user namespaces, use NIX_TEST_FORCE_NETWORK_TESTS=1 to override";
|
||||
} else {
|
||||
GTEST_SKIP();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace nix::testing
|
||||
@@ -4,8 +4,6 @@ include_dirs = [ include_directories('../../..') ]
|
||||
|
||||
headers = files(
|
||||
'derived-path.hh',
|
||||
'https-store.hh',
|
||||
'libstore-network.hh',
|
||||
'libstore.hh',
|
||||
'nix_api_store.hh',
|
||||
'outputs-spec.hh',
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
#include "nix/store/tests/libstore-network.hh"
|
||||
#include "nix/util/error.hh"
|
||||
#include "nix/util/environment-variables.hh"
|
||||
|
||||
#ifdef __linux__
|
||||
# include "nix/util/file-system.hh"
|
||||
# include "nix/util/linux-namespaces.hh"
|
||||
# include <sched.h>
|
||||
# include <sys/ioctl.h>
|
||||
# include <net/if.h>
|
||||
# include <netinet/in.h>
|
||||
#endif
|
||||
|
||||
namespace nix::testing {
|
||||
|
||||
bool networkTestsAvailable = false;
|
||||
|
||||
#ifdef __linux__
|
||||
|
||||
static void enterNetworkNamespace()
|
||||
{
|
||||
auto uid = ::getuid();
|
||||
auto gid = ::getgid();
|
||||
|
||||
if (::unshare(CLONE_NEWUSER | CLONE_NEWNET) == -1)
|
||||
throw SysError("setting up a private network namespace for tests");
|
||||
|
||||
std::filesystem::path procSelf = "/proc/self";
|
||||
writeFile(procSelf / "setgroups", "deny");
|
||||
writeFile(procSelf / "uid_map", fmt("%d %d 1", uid, uid));
|
||||
writeFile(procSelf / "gid_map", fmt("%d %d 1", gid, gid));
|
||||
|
||||
AutoCloseFD fd(::socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
|
||||
if (!fd)
|
||||
throw SysError("cannot open IP socket for loopback interface");
|
||||
|
||||
struct ::ifreq ifr = {};
|
||||
strcpy(ifr.ifr_name, "lo");
|
||||
ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
|
||||
if (::ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
|
||||
throw SysError("cannot set loopback interface flags");
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void setupNetworkTests()
|
||||
try {
|
||||
networkTestsAvailable = getEnvOs(OS_STR("NIX_TEST_FORCE_NETWORK_TESTS")).has_value();
|
||||
|
||||
#ifdef __linux__
|
||||
if (!networkTestsAvailable && userNamespacesSupported()) {
|
||||
enterNetworkNamespace();
|
||||
networkTestsAvailable = true;
|
||||
}
|
||||
#endif
|
||||
} catch (SystemError & e) {
|
||||
/* Ignore any set up errors. */
|
||||
}
|
||||
|
||||
} // namespace nix::testing
|
||||
@@ -28,15 +28,10 @@ subdir('nix-meson-build-support/subprojects')
|
||||
rapidcheck = dependency('rapidcheck')
|
||||
deps_public += rapidcheck
|
||||
|
||||
gtest = dependency('gtest')
|
||||
deps_public += gtest
|
||||
|
||||
subdir('nix-meson-build-support/common')
|
||||
|
||||
sources = files(
|
||||
'derived-path.cc',
|
||||
'https-store.cc',
|
||||
'libstore-network.cc',
|
||||
'outputs-spec.cc',
|
||||
'path.cc',
|
||||
'test-main.cc',
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
nix-store-c,
|
||||
|
||||
rapidcheck,
|
||||
gtest,
|
||||
|
||||
# Configuration Options
|
||||
|
||||
@@ -40,7 +39,6 @@ mkMesonLibrary (finalAttrs: {
|
||||
nix-store
|
||||
nix-store-c
|
||||
rapidcheck
|
||||
gtest
|
||||
];
|
||||
|
||||
mesonFlags = [
|
||||
|
||||
@@ -138,11 +138,7 @@ CHARACTERIZATION_TEST(
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
@@ -158,11 +154,7 @@ READ_CHARACTERIZATION_TEST(
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
Binary file not shown.
@@ -6,8 +6,8 @@
|
||||
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
]
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -10,8 +10,8 @@
|
||||
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -12,8 +12,8 @@
|
||||
],
|
||||
"registrationTime": 23423,
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": true,
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
],
|
||||
"registrationTime": 23423,
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": true,
|
||||
|
||||
@@ -9,8 +9,8 @@
|
||||
],
|
||||
"registrationTime": 23423,
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": true,
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
],
|
||||
"registrationTime": 23423,
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": true,
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdfasdfasdf"
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@@ -6,8 +6,8 @@
|
||||
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
]
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -10,8 +10,8 @@
|
||||
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
]
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -27,8 +27,8 @@
|
||||
],
|
||||
"registrationTime": null,
|
||||
"signatures": [
|
||||
"fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"fake-sig-1",
|
||||
"fake-sig-2"
|
||||
],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
|
||||
Binary file not shown.
@@ -6,8 +6,8 @@
|
||||
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
]
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -10,8 +10,8 @@
|
||||
"id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"asdf",
|
||||
"qwer"
|
||||
]
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -24,8 +24,8 @@
|
||||
],
|
||||
"registrationTime": 23423,
|
||||
"signatures": [
|
||||
"fake-sig-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"fake-sig-2:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
"fake-sig-1",
|
||||
"fake-sig-2"
|
||||
],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <regex>
|
||||
|
||||
#include "nix/store/http-binary-cache-store.hh"
|
||||
#include "nix/store/tests/https-store.hh"
|
||||
#include "nix/util/fs-sink.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -37,88 +34,4 @@ TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams)
|
||||
EXPECT_EQ(config.getReference().params, params);
|
||||
}
|
||||
|
||||
using testing::HttpsBinaryCacheStoreMtlsTest;
|
||||
using testing::HttpsBinaryCacheStoreTest;
|
||||
|
||||
using namespace std::string_view_literals;
|
||||
using namespace std::string_literals;
|
||||
|
||||
TEST_F(HttpsBinaryCacheStoreTest, queryPathInfo)
|
||||
{
|
||||
auto config = makeConfig({});
|
||||
auto store = config->openStore();
|
||||
StringSource dump{"test"sv};
|
||||
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
|
||||
EXPECT_NO_THROW(store->queryPathInfo(path));
|
||||
}
|
||||
|
||||
auto withNoRetries()
|
||||
{
|
||||
auto oldTries = fileTransferSettings.tries.get();
|
||||
Finally restoreTries = [=]() { fileTransferSettings.tries = oldTries; };
|
||||
fileTransferSettings.tries = 1; /* FIXME: Don't use global settings. */
|
||||
return restoreTries;
|
||||
}
|
||||
|
||||
TEST_F(HttpsBinaryCacheStoreMtlsTest, queryPathInfo)
|
||||
{
|
||||
auto config = makeConfig({
|
||||
{"tls-certificate"s, clientCert.string()},
|
||||
{"tls-private-key"s, clientKey.string()},
|
||||
});
|
||||
auto store = config->openStore();
|
||||
StringSource dump{"test"sv};
|
||||
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
|
||||
EXPECT_NO_THROW(store->queryPathInfo(path));
|
||||
}
|
||||
|
||||
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWithoutClientCert)
|
||||
{
|
||||
auto restoreTries = withNoRetries();
|
||||
auto config = makeConfig({});
|
||||
EXPECT_THROW(config->openStore(), Error);
|
||||
}
|
||||
|
||||
TEST_F(HttpsBinaryCacheStoreMtlsTest, rejectsWrongClientCert)
|
||||
{
|
||||
auto wrongKey = tmpDir / "wrong.key";
|
||||
auto wrongCert = tmpDir / "wrong.crt";
|
||||
|
||||
// clang-format off
|
||||
openssl({"ecparam", "-genkey", "-name", "prime256v1", "-out", wrongKey.string()});
|
||||
openssl({"req", "-new", "-x509", "-days", "1", "-key", wrongKey.string(), "-out", wrongCert.string(), "-subj", "/CN=WrongClient"});
|
||||
// clang-format on
|
||||
|
||||
auto config = makeConfig({
|
||||
{"tls-certificate"s, wrongCert.string()},
|
||||
{"tls-private-key"s, wrongKey.string()},
|
||||
});
|
||||
auto restoreTries = withNoRetries();
|
||||
EXPECT_THROW(config->openStore(), Error);
|
||||
}
|
||||
|
||||
TEST_F(HttpsBinaryCacheStoreMtlsTest, doesNotSendCertOnRedirectToDifferentAuthority)
|
||||
{
|
||||
StringSource dump{"test"sv};
|
||||
auto path = localCacheStore->addToStoreFromDump(dump, "test-name", FileSerialisationMethod::Flat);
|
||||
|
||||
for (auto & entry : DirectoryIterator{cacheDir})
|
||||
if (entry.path().extension() == ".narinfo") {
|
||||
auto content = readFile(entry.path());
|
||||
content = std::regex_replace(content, std::regex("URL: nar/"), fmt("URL: https://127.0.0.1:%d/nar/", port));
|
||||
writeFile(entry.path(), content);
|
||||
}
|
||||
|
||||
auto config = makeConfig({
|
||||
{"tls-certificate"s, clientCert.string()},
|
||||
{"tls-private-key"s, clientKey.string()},
|
||||
});
|
||||
auto store = config->openStore();
|
||||
|
||||
auto restoreTries = withNoRetries();
|
||||
auto info = store->queryPathInfo(path);
|
||||
NullSink null;
|
||||
EXPECT_THROW(store->narFromPath(path, null), Error);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -1,7 +1,6 @@
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "nix/store/tests/test-main.hh"
|
||||
#include "nix/store/tests/libstore-network.hh"
|
||||
|
||||
using namespace nix;
|
||||
|
||||
@@ -11,7 +10,6 @@ int main(int argc, char ** argv)
|
||||
if (res)
|
||||
return res;
|
||||
|
||||
nix::testing::setupNetworkTests();
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <rapidcheck/gtest.h>
|
||||
#include "nix/store/globals.hh"
|
||||
#include "nix/store/sqlite.hh"
|
||||
#include <sqlite3.h>
|
||||
|
||||
@@ -49,7 +48,7 @@ TEST(NarInfoDiskCacheImpl, create_and_read)
|
||||
|
||||
// We're going to pay special attention to the id field because we had a bug
|
||||
// that changed it.
|
||||
db = SQLite(dbPath, {.useWAL = settings.useSQLiteWAL});
|
||||
db = SQLite(dbPath);
|
||||
getIds.create(db, "select id from BinaryCaches where url = 'http://foo'");
|
||||
|
||||
{
|
||||
|
||||
@@ -59,10 +59,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo)
|
||||
};
|
||||
info.registrationTime = 23423;
|
||||
info.ultimate = true;
|
||||
info.sigs = {
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
};
|
||||
info.sigs = {"asdf", "qwer"};
|
||||
|
||||
info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz";
|
||||
info.compression = "xz";
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
nix-store-c,
|
||||
nix-store-test-support,
|
||||
sqlite,
|
||||
openssl,
|
||||
|
||||
rapidcheck,
|
||||
gtest,
|
||||
@@ -76,10 +75,7 @@ mkMesonExecutable (finalAttrs: {
|
||||
runCommand "${finalAttrs.pname}-run"
|
||||
{
|
||||
meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
|
||||
nativeBuildInputs = [
|
||||
writableTmpDirAsHomeHook
|
||||
openssl
|
||||
];
|
||||
buildInputs = [ writableTmpDirAsHomeHook ];
|
||||
}
|
||||
(
|
||||
''
|
||||
|
||||
@@ -66,10 +66,7 @@ static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo)
|
||||
};
|
||||
info.registrationTime = 23423;
|
||||
info.ultimate = true;
|
||||
info.sigs = {
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
};
|
||||
info.sigs = {"asdf", "qwer"};
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
@@ -70,9 +70,7 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
[&] {
|
||||
auto r = simple;
|
||||
// FIXME actually sign properly
|
||||
r.signatures = {
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
};
|
||||
r.signatures = {"asdfasdfasdf"};
|
||||
return r;
|
||||
}(),
|
||||
}));
|
||||
|
||||
@@ -109,11 +109,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
@@ -131,11 +127,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
@@ -329,8 +321,8 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
info.narSize = 34878;
|
||||
info.sigs =
|
||||
{
|
||||
Signature{.keyName = "fake-sig-1", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "fake-sig-2", .sig = std::string(64, '\0')},
|
||||
"fake-sig-1",
|
||||
"fake-sig-2",
|
||||
},
|
||||
static_cast<UnkeyedValidPathInfo>(std::move(info));
|
||||
}),
|
||||
|
||||
@@ -162,11 +162,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
@@ -184,11 +180,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
@@ -549,8 +541,8 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
info.narSize = 34878;
|
||||
info.sigs =
|
||||
{
|
||||
Signature{.keyName = "fake-sig-1", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "fake-sig-2", .sig = std::string(64, '\0')},
|
||||
"fake-sig-1",
|
||||
"fake-sig-2",
|
||||
},
|
||||
info;
|
||||
}),
|
||||
|
||||
+12
-103
@@ -13,11 +13,6 @@
|
||||
// C library headers for SSO provider support
|
||||
# include <aws/auth/credentials.h>
|
||||
|
||||
// C library headers for custom logging
|
||||
# include <aws/common/logging.h>
|
||||
|
||||
# include <cstdarg>
|
||||
|
||||
# include <boost/unordered/concurrent_flat_map.hpp>
|
||||
|
||||
# include <chrono>
|
||||
@@ -35,101 +30,6 @@ AwsAuthError::AwsAuthError(int errorCode)
|
||||
|
||||
namespace {
|
||||
|
||||
/**
|
||||
* Map AWS log level to Nix verbosity.
|
||||
* AWS levels: AWS_LL_NONE=0, AWS_LL_FATAL=1, AWS_LL_ERROR=2, AWS_LL_WARN=3,
|
||||
* AWS_LL_INFO=4, AWS_LL_DEBUG=5, AWS_LL_TRACE=6
|
||||
*
|
||||
* We map very conservatively because the AWS SDK is extremely noisy. What AWS
|
||||
* considers "info" includes low-level details like "Initializing epoll" and
|
||||
* "Starting event-loop thread". What it considers "errors" includes expected
|
||||
* conditions like missing ~/.aws/config or IMDS being unavailable on non-EC2.
|
||||
*
|
||||
* To avoid spamming users, we only show FATAL at default verbosity. Everything
|
||||
* else requires -vvvvv (lvlDebug) or higher to see.
|
||||
*/
|
||||
static Verbosity awsLogLevelToVerbosity(enum aws_log_level level)
|
||||
{
|
||||
switch (level) {
|
||||
case AWS_LL_FATAL:
|
||||
return lvlError;
|
||||
case AWS_LL_ERROR:
|
||||
case AWS_LL_WARN:
|
||||
case AWS_LL_INFO:
|
||||
return lvlDebug;
|
||||
case AWS_LL_DEBUG:
|
||||
case AWS_LL_TRACE:
|
||||
return lvlVomit;
|
||||
// AWS_LL_NONE and AWS_LL_COUNT are enum sentinels, not real log levels
|
||||
case AWS_LL_NONE:
|
||||
case AWS_LL_COUNT:
|
||||
return lvlDebug;
|
||||
}
|
||||
unreachable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom AWS logger that routes logs through Nix's logging infrastructure.
|
||||
*
|
||||
* The AWS CRT C++ wrapper (ApiHandle::InitializeLogging) only supports FILE*
|
||||
* or filename-based logging. The underlying C library supports custom loggers
|
||||
* via aws_logger struct with a vtable containing callback functions.
|
||||
*/
|
||||
static int nixAwsLoggerLog(
|
||||
struct aws_logger * logger, enum aws_log_level logLevel, aws_log_subject_t subject, const char * format, ...)
|
||||
{
|
||||
Verbosity nixLevel = awsLogLevelToVerbosity(logLevel);
|
||||
if (nixLevel > verbosity)
|
||||
return AWS_OP_SUCCESS; /* Bail out early to avoid formatting the message unnecessarily. */
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
std::array<char, 4096> buffer{};
|
||||
auto res = vsnprintf(buffer.data(), buffer.size(), format, args);
|
||||
va_end(args);
|
||||
if (res < 0) /* Skip garbage debug messages in case the SDK is busted. */
|
||||
return AWS_OP_SUCCESS;
|
||||
|
||||
const char * subjectName = aws_log_subject_name(subject);
|
||||
printMsgUsing(nix::logger, nixLevel, "(aws:%s) %s", subjectName ? subjectName : "unknown", chomp(buffer.data()));
|
||||
return AWS_OP_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current log level for a subject - determines which messages will be logged.
|
||||
* Must be consistent with awsLogLevelToVerbosity mapping.
|
||||
*/
|
||||
static aws_log_level nixAwsLoggerGetLevel(struct aws_logger * logger, aws_log_subject_t subject)
|
||||
{
|
||||
// Map Nix verbosity back to AWS log level (inverse of awsLogLevelToVerbosity)
|
||||
if (verbosity >= lvlVomit)
|
||||
return AWS_LL_TRACE;
|
||||
if (verbosity >= lvlDebug)
|
||||
return AWS_LL_INFO; // error/warn/info are all mapped to lvlDebug
|
||||
return AWS_LL_FATAL;
|
||||
}
|
||||
|
||||
static void initialiseAwsLogger()
|
||||
{
|
||||
static std::once_flag initialised; /* aws_logger_set must only be called once */
|
||||
std::call_once(initialised, []() {
|
||||
static aws_logger_vtable nixAwsLoggerVtable = {
|
||||
.log = nixAwsLoggerLog,
|
||||
.get_log_level = nixAwsLoggerGetLevel,
|
||||
.clean_up = [](struct aws_logger *) {}, // No resources to clean up
|
||||
.set_log_level = nullptr,
|
||||
};
|
||||
|
||||
static aws_logger nixAwsLogger = {
|
||||
.vtable = &nixAwsLoggerVtable,
|
||||
.allocator = nullptr,
|
||||
.p_impl = nullptr,
|
||||
};
|
||||
|
||||
aws_logger_set(&nixAwsLogger);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to wrap a C credentials provider in the C++ interface.
|
||||
* This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
|
||||
@@ -219,9 +119,18 @@ class AwsCredentialProviderImpl : public AwsCredentialProvider
|
||||
public:
|
||||
AwsCredentialProviderImpl()
|
||||
{
|
||||
// Install custom logger that routes AWS CRT logs through Nix's logging infrastructure.
|
||||
// This ensures AWS logs respect Nix's verbosity settings and are formatted consistently.
|
||||
initialiseAwsLogger();
|
||||
// Map Nix's verbosity to AWS CRT log level
|
||||
Aws::Crt::LogLevel logLevel;
|
||||
if (verbosity >= lvlVomit) {
|
||||
logLevel = Aws::Crt::LogLevel::Trace;
|
||||
} else if (verbosity >= lvlDebug) {
|
||||
logLevel = Aws::Crt::LogLevel::Debug;
|
||||
} else if (verbosity >= lvlChatty) {
|
||||
logLevel = Aws::Crt::LogLevel::Info;
|
||||
} else {
|
||||
logLevel = Aws::Crt::LogLevel::Warn;
|
||||
}
|
||||
apiHandle.InitializeLogging(logLevel, stderr);
|
||||
|
||||
// Create a shared TLS context for SSO (required for HTTPS connections)
|
||||
auto allocator = Aws::Crt::ApiAllocator();
|
||||
|
||||
@@ -147,7 +147,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
write the compressed NAR to disk), into a HashSink (to get the
|
||||
NAR hash), and into a NarAccessor (to get the NAR listing). */
|
||||
HashSink fileHashSink{HashAlgorithm::SHA256};
|
||||
std::shared_ptr<NarAccessor> narAccessor;
|
||||
std::shared_ptr<SourceAccessor> narAccessor;
|
||||
HashSink narHashSink{HashAlgorithm::SHA256};
|
||||
{
|
||||
FdSink fileSink(fdTemp.get());
|
||||
@@ -156,7 +156,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
config.compression, teeSinkCompressed, config.parallelCompression, config.compressionLevel);
|
||||
TeeSink teeSinkUncompressed{*compressionSink, narHashSink};
|
||||
TeeSource teeSource{narSource, teeSinkUncompressed};
|
||||
narAccessor = makeNarAccessor(parseNarListing(teeSource));
|
||||
narAccessor = makeNarAccessor(teeSource);
|
||||
compressionSink->finish();
|
||||
fileSink.flush();
|
||||
}
|
||||
@@ -165,18 +165,18 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
|
||||
auto info = mkInfo(narHashSink.finish());
|
||||
auto narInfo = make_ref<NarInfo>(info);
|
||||
narInfo->compression = config.compression.to_string(); // FIXME: Make NarInfo use CompressionAlgo
|
||||
narInfo->compression = config.compression;
|
||||
auto [fileHash, fileSize] = fileHashSink.finish();
|
||||
narInfo->fileHash = fileHash;
|
||||
narInfo->fileSize = fileSize;
|
||||
narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar"
|
||||
+ (config.compression == CompressionAlgo::xz ? ".xz"
|
||||
: config.compression == CompressionAlgo::bzip2 ? ".bz2"
|
||||
: config.compression == CompressionAlgo::zstd ? ".zst"
|
||||
: config.compression == CompressionAlgo::lzip ? ".lzip"
|
||||
: config.compression == CompressionAlgo::lz4 ? ".lz4"
|
||||
: config.compression == CompressionAlgo::brotli ? ".br"
|
||||
: "");
|
||||
+ (config.compression == "xz" ? ".xz"
|
||||
: config.compression == "bzip2" ? ".bz2"
|
||||
: config.compression == "zstd" ? ".zst"
|
||||
: config.compression == "lzip" ? ".lzip"
|
||||
: config.compression == "lz4" ? ".lz4"
|
||||
: config.compression == "br" ? ".br"
|
||||
: "");
|
||||
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
||||
printMsg(
|
||||
@@ -205,7 +205,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
|
||||
if (config.writeNARListing) {
|
||||
nlohmann::json j = {
|
||||
{"version", 1},
|
||||
{"root", narAccessor->getListing()},
|
||||
{"root", listNarDeep(*narAccessor, CanonPath::root)},
|
||||
};
|
||||
|
||||
upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");
|
||||
@@ -562,10 +562,10 @@ ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
|
||||
|
||||
std::shared_ptr<SourceAccessor> BinaryCacheStore::getFSAccessor(const StorePath & storePath, bool requireValidPath)
|
||||
{
|
||||
return static_cast<ref<SourceAccessor>>(getRemoteFSAccessor(requireValidPath)->accessObject(storePath));
|
||||
return getRemoteFSAccessor(requireValidPath)->accessObject(storePath);
|
||||
}
|
||||
|
||||
void BinaryCacheStore::addSignatures(const StorePath & storePath, const std::set<Signature> & sigs)
|
||||
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
|
||||
{
|
||||
/* Note: this is inherently racy since there is no locking on
|
||||
binary caches. In particular, with S3 this unreliable, even
|
||||
|
||||
@@ -75,11 +75,6 @@ static BuildResult::Failure::Status failureStatusFromString(std::string_view str
|
||||
throw Error("unknown built result failure status '%s'", str);
|
||||
}
|
||||
|
||||
[[noreturn]] void BuildResult::Failure::rethrow() const
|
||||
{
|
||||
throw BuildError(status, "%s", errorMsg);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
namespace nlohmann {
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
#include "nix/store/build/build-log.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
BuildLog::BuildLog(size_t maxTailLines, std::unique_ptr<Activity> act)
|
||||
: maxTailLines(maxTailLines)
|
||||
, act(std::move(act))
|
||||
{
|
||||
}
|
||||
|
||||
void BuildLog::operator()(std::string_view data)
|
||||
{
|
||||
for (auto c : data)
|
||||
if (c == '\r')
|
||||
currentLogLinePos = 0;
|
||||
else if (c == '\n')
|
||||
flushLine();
|
||||
else {
|
||||
if (currentLogLinePos >= currentLogLine.size())
|
||||
currentLogLine.resize(currentLogLinePos + 1);
|
||||
currentLogLine[currentLogLinePos++] = c;
|
||||
}
|
||||
}
|
||||
|
||||
void BuildLog::flush()
|
||||
{
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
}
|
||||
|
||||
void BuildLog::flushLine()
|
||||
{
|
||||
// Truncate to actual content (currentLogLinePos may be less than size due to \r)
|
||||
currentLogLine.resize(currentLogLinePos);
|
||||
|
||||
if (!handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) {
|
||||
// Line was not handled as JSON, emit and add to tail
|
||||
act->result(resBuildLogLine, currentLogLine);
|
||||
logTail.push_back(currentLogLine);
|
||||
if (logTail.size() > maxTailLines)
|
||||
logTail.pop_front();
|
||||
}
|
||||
|
||||
currentLogLine.clear();
|
||||
currentLogLinePos = 0;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -40,13 +40,37 @@ DerivationBuildingGoal::DerivationBuildingGoal(
|
||||
worker.store.addTempRoot(this->drvPath);
|
||||
}
|
||||
|
||||
DerivationBuildingGoal::~DerivationBuildingGoal() = default;
|
||||
DerivationBuildingGoal::~DerivationBuildingGoal()
|
||||
{
|
||||
/* Careful: we should never ever throw an exception from a
|
||||
destructor. */
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
if (builder)
|
||||
builder.reset();
|
||||
#endif
|
||||
try {
|
||||
closeLogFile();
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
}
|
||||
}
|
||||
|
||||
std::string DerivationBuildingGoal::key()
|
||||
{
|
||||
return "dd$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath);
|
||||
}
|
||||
|
||||
void DerivationBuildingGoal::killChild()
|
||||
{
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook.reset();
|
||||
#endif
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
if (builder && builder->killChild())
|
||||
worker.childTerminated(this);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
|
||||
{
|
||||
std::string msg;
|
||||
@@ -122,8 +146,6 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
|
||||
writeDerivation(worker.store, *drv);
|
||||
}
|
||||
|
||||
StorePathSet inputPaths;
|
||||
|
||||
{
|
||||
/* If we get this far, we know no dynamic drvs inputs */
|
||||
|
||||
@@ -163,23 +185,10 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
|
||||
slot to become available, since we don't need one if there is a
|
||||
build hook. */
|
||||
co_await yield();
|
||||
co_return tryToBuild(std::move(inputPaths));
|
||||
co_return tryToBuild();
|
||||
}
|
||||
|
||||
/**
|
||||
* RAII wrapper for build log file.
|
||||
* Constructor opens the log file, destructor closes it.
|
||||
*/
|
||||
struct LogFile
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
std::shared_ptr<BufferedSink> fileSink, sink;
|
||||
|
||||
LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings);
|
||||
~LogFile();
|
||||
};
|
||||
|
||||
Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
Goal::Co DerivationBuildingGoal::tryToBuild()
|
||||
{
|
||||
auto drvOptions = [&] {
|
||||
DerivationOptions<SingleDerivedPath> temp;
|
||||
@@ -240,6 +249,33 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
}
|
||||
checkPathValidity(initialOutputs);
|
||||
|
||||
auto started = [&]() {
|
||||
auto msg =
|
||||
fmt(buildMode == bmRepair ? "repairing outputs of '%s'"
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook)
|
||||
msg += fmt(" on '%s'", hook->machineName);
|
||||
#endif
|
||||
act = std::make_unique<Activity>(
|
||||
*logger,
|
||||
lvlInfo,
|
||||
actBuild,
|
||||
msg,
|
||||
Logger::Fields{
|
||||
worker.store.printStorePath(drvPath),
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook ? hook->machineName :
|
||||
#endif
|
||||
"",
|
||||
1,
|
||||
1});
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
};
|
||||
|
||||
/**
|
||||
* Activity that denotes waiting for a lock.
|
||||
*/
|
||||
@@ -330,7 +366,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
if (buildLocally) {
|
||||
useHook = false;
|
||||
} else {
|
||||
switch (tryBuildHook(drvOptions)) {
|
||||
switch (tryBuildHook(initialOutputs, drvOptions)) {
|
||||
case rpAccept:
|
||||
/* Yes, it has started doing so. Wait until we get
|
||||
EOF from the hook. */
|
||||
@@ -380,6 +416,9 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
Magenta(
|
||||
"/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon"));
|
||||
|
||||
#ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
builder.reset();
|
||||
#endif
|
||||
outputLocks.unlock();
|
||||
worker.permanentFailure = true;
|
||||
co_return doneFailure({BuildResult::Failure::InputRejected, std::move(msg)});
|
||||
@@ -394,229 +433,103 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
actLock.reset();
|
||||
|
||||
if (useHook) {
|
||||
co_return buildWithHook(
|
||||
std::move(inputPaths), std::move(initialOutputs), std::move(drvOptions), std::move(outputLocks));
|
||||
} else {
|
||||
co_return buildLocally(
|
||||
std::move(inputPaths),
|
||||
std::move(initialOutputs),
|
||||
std::move(drvOptions),
|
||||
std::move(outputLocks),
|
||||
externalBuilder);
|
||||
}
|
||||
}
|
||||
buildResult.startTime = time(0); // inexact
|
||||
started();
|
||||
|
||||
Goal::Co DerivationBuildingGoal::buildWithHook(
|
||||
StorePathSet inputPaths,
|
||||
std::map<std::string, InitialOutput> initialOutputs,
|
||||
DerivationOptions<StorePath> drvOptions,
|
||||
PathLocks outputLocks)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
unreachable();
|
||||
#else
|
||||
std::unique_ptr<HookInstance> hook = std::move(worker.hook);
|
||||
|
||||
/* Set up callback so childTerminated is called if the hook is
|
||||
destroyed (e.g., during failure cascades). */
|
||||
hook->onKillChild = [this]() { worker.childTerminated(this, JobCategory::Build); };
|
||||
|
||||
try {
|
||||
hook->machineName = readLine(hook->fromHook.readSide.get());
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while reading the machine name from the build hook");
|
||||
throw;
|
||||
}
|
||||
|
||||
CommonProto::WriteConn conn{hook->sink};
|
||||
|
||||
/* Tell the hook all the inputs that have to be copied to the
|
||||
remote system. */
|
||||
CommonProto::write(worker.store, conn, inputPaths);
|
||||
|
||||
/* Tell the hooks the missing outputs that have to be copied back
|
||||
from the remote system. */
|
||||
{
|
||||
StringSet missingOutputs;
|
||||
for (auto & [outputName, status] : initialOutputs) {
|
||||
// XXX: Does this include known CA outputs?
|
||||
if (buildMode != bmCheck && status.known && status.known->isValid())
|
||||
continue;
|
||||
missingOutputs.insert(outputName);
|
||||
}
|
||||
CommonProto::write(worker.store, conn, missingOutputs);
|
||||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
hook->toHook.writeSide.close();
|
||||
|
||||
/* Create the log file and pipe. */
|
||||
std::unique_ptr<LogFile> logFile = std::make_unique<LogFile>(worker.store, drvPath, settings.getLogFileSettings());
|
||||
|
||||
std::set<MuxablePipePollState::CommChannel> fds;
|
||||
fds.insert(hook->fromHook.readSide.get());
|
||||
fds.insert(hook->builderOut.readSide.get());
|
||||
worker.childStarted(shared_from_this(), fds, false, false);
|
||||
|
||||
buildResult.startTime = time(0); // inexact
|
||||
|
||||
auto msg =
|
||||
fmt(buildMode == bmRepair ? "repairing outputs of '%s'"
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
msg += fmt(" on '%s'", hook->machineName);
|
||||
|
||||
std::unique_ptr<BuildLog> buildLog = std::make_unique<BuildLog>(
|
||||
settings.logLines,
|
||||
std::make_unique<Activity>(
|
||||
*logger,
|
||||
lvlInfo,
|
||||
actBuild,
|
||||
msg,
|
||||
Logger::Fields{worker.store.printStorePath(drvPath), hook->machineName, 1, 1}));
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
|
||||
std::string currentHookLine;
|
||||
uint64_t logSize = 0;
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
auto & fd = output->fd;
|
||||
auto & data = output->data;
|
||||
if (fd == hook->builderOut.readSide.get()) {
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
hook.reset();
|
||||
co_return doneFailureLogTooLong(*buildLog);
|
||||
}
|
||||
(*buildLog)(data);
|
||||
if (logFile->sink)
|
||||
(*logFile->sink)(data);
|
||||
} else if (fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
auto json = parseJSONMessage(currentHookLine, "the derivation builder");
|
||||
if (json) {
|
||||
auto s = handleJSONLogMessage(
|
||||
*json, worker.act, hook->activities, "the derivation builder", true);
|
||||
// ensure that logs from a builder using `ssh-ng://` as protocol
|
||||
// are also available to `nix log`.
|
||||
if (s && logFile->sink) {
|
||||
const auto type = (*json)["type"];
|
||||
const auto fields = (*json)["fields"];
|
||||
if (type == resBuildLogLine) {
|
||||
(*logFile->sink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
|
||||
} else if (type == resSetPhase && !fields.is_null()) {
|
||||
const auto phase = fields[0];
|
||||
if (!phase.is_null()) {
|
||||
// nixpkgs' stdenv produces lines in the log to signal
|
||||
// phase changes.
|
||||
// We want to get the same lines in case of remote builds.
|
||||
// The format is:
|
||||
// @nix { "action": "setPhase", "phase": "$curPhase" }
|
||||
const auto logLine =
|
||||
nlohmann::json::object({{"action", "setPhase"}, {"phase", phase}});
|
||||
(*logFile->sink)(
|
||||
"@nix "
|
||||
+ logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)
|
||||
+ "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
currentHookLine.clear();
|
||||
} else
|
||||
currentHookLine += c;
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
co_await processChildOutput(output->fd, output->data);
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
killChild();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
buildLog->flush();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
hook.reset();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
|
||||
trace("hook build done");
|
||||
#ifndef _WIN32
|
||||
assert(hook);
|
||||
#endif
|
||||
|
||||
/* Since we got an EOF on the logger pipe, the builder is presumed
|
||||
to have terminated. In fact, the builder could also have
|
||||
simply have closed its end of the pipe, so just to be sure,
|
||||
kill it. */
|
||||
int status = hook->pid.kill();
|
||||
trace("hook build done");
|
||||
|
||||
debug("build hook for '%s' finished", worker.store.printStorePath(drvPath));
|
||||
/* Since we got an EOF on the logger pipe, the builder is presumed
|
||||
to have terminated. In fact, the builder could also have
|
||||
simply have closed its end of the pipe, so just to be sure,
|
||||
kill it. */
|
||||
int status =
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook->pid.kill();
|
||||
#else
|
||||
0;
|
||||
#endif
|
||||
|
||||
buildResult.timesBuilt++;
|
||||
buildResult.stopTime = time(0);
|
||||
debug("build hook for '%s' finished", worker.store.printStorePath(drvPath));
|
||||
|
||||
/* So the child is gone now. */
|
||||
worker.childTerminated(this);
|
||||
buildResult.timesBuilt++;
|
||||
buildResult.stopTime = time(0);
|
||||
|
||||
/* Close the read side of the logger pipe. */
|
||||
hook->builderOut.readSide.close();
|
||||
hook->fromHook.readSide.close();
|
||||
/* So the child is gone now. */
|
||||
worker.childTerminated(this);
|
||||
|
||||
/* Close the log file. */
|
||||
logFile.reset();
|
||||
/* Close the read side of the logger pipe. */
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook->builderOut.readSide.close();
|
||||
hook->fromHook.readSide.close();
|
||||
#endif
|
||||
|
||||
/* Check the exit status. */
|
||||
if (!statusOk(status)) {
|
||||
auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""}, *buildLog);
|
||||
/* Close the log file. */
|
||||
closeLogFile();
|
||||
|
||||
/* Check the exit status. */
|
||||
if (!statusOk(status)) {
|
||||
auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""});
|
||||
|
||||
outputLocks.unlock();
|
||||
|
||||
/* TODO (once again) support fine-grained error codes, see issue #12641. */
|
||||
|
||||
co_return doneFailure(std::move(e));
|
||||
}
|
||||
|
||||
/* Compute the FS closure of the outputs and register them as
|
||||
being valid. */
|
||||
auto builtOutputs =
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
to do anything here.
|
||||
|
||||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressing derivations this isn't the case.
|
||||
|
||||
Aborts if any output is not valid or corrupt, and otherwise
|
||||
returns a 'SingleDrvOutputs' structure containing all outputs.
|
||||
*/
|
||||
[&] {
|
||||
auto [allValid, validOutputs] = checkPathValidity(initialOutputs);
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
}();
|
||||
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, output] : builtOutputs)
|
||||
outputPaths.insert(output.outPath);
|
||||
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
|
||||
|
||||
/* It is now safe to delete the lock files, since all future
|
||||
lockers will see that the output paths are valid; they will
|
||||
not create new lock files with the same names as the old
|
||||
(unlinked) lock files. */
|
||||
outputLocks.setDeletion(true);
|
||||
outputLocks.unlock();
|
||||
|
||||
/* TODO (once again) support fine-grained error codes, see issue #12641. */
|
||||
|
||||
co_return doneFailure(std::move(e));
|
||||
co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs));
|
||||
}
|
||||
|
||||
/* Compute the FS closure of the outputs and register them as
|
||||
being valid. */
|
||||
auto builtOutputs =
|
||||
/* When using a build hook, the build hook can register the output
|
||||
as valid (by doing `nix-store --import'). If so we don't have
|
||||
to do anything here.
|
||||
|
||||
We can only early return when the outputs are known a priori. For
|
||||
floating content-addressing derivations this isn't the case.
|
||||
|
||||
Aborts if any output is not valid or corrupt, and otherwise
|
||||
returns a 'SingleDrvOutputs' structure containing all outputs.
|
||||
*/
|
||||
[&] {
|
||||
auto [allValid, validOutputs] = checkPathValidity(initialOutputs);
|
||||
if (!allValid)
|
||||
throw Error("some outputs are unexpectedly invalid");
|
||||
return validOutputs;
|
||||
}();
|
||||
|
||||
StorePathSet outputPaths;
|
||||
for (auto & [_, output] : builtOutputs)
|
||||
outputPaths.insert(output.outPath);
|
||||
runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
|
||||
|
||||
/* It is now safe to delete the lock files, since all future
|
||||
lockers will see that the output paths are valid; they will
|
||||
not create new lock files with the same names as the old
|
||||
(unlinked) lock files. */
|
||||
outputLocks.setDeletion(true);
|
||||
outputLocks.unlock();
|
||||
|
||||
co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs));
|
||||
#endif
|
||||
}
|
||||
|
||||
Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
StorePathSet inputPaths,
|
||||
std::map<std::string, InitialOutput> initialOutputs,
|
||||
DerivationOptions<StorePath> drvOptions,
|
||||
PathLocks outputLocks,
|
||||
const ExternalBuilder * externalBuilder)
|
||||
{
|
||||
co_await yield();
|
||||
|
||||
if (!dynamic_cast<LocalStore *>(&worker.store)) {
|
||||
@@ -632,31 +545,8 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
#ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows
|
||||
throw UnimplementedError("building derivations is not yet implemented on Windows");
|
||||
#else
|
||||
std::unique_ptr<BuildLog> buildLog;
|
||||
std::unique_ptr<LogFile> logFile;
|
||||
assert(!hook);
|
||||
|
||||
auto openLogFile = [&]() {
|
||||
logFile = std::make_unique<LogFile>(worker.store, drvPath, settings.getLogFileSettings());
|
||||
};
|
||||
|
||||
auto closeLogFile = [&]() { logFile.reset(); };
|
||||
|
||||
auto started = [&]() {
|
||||
auto msg =
|
||||
fmt(buildMode == bmRepair ? "repairing outputs of '%s'"
|
||||
: buildMode == bmCheck ? "checking outputs of '%s'"
|
||||
: "building '%s'",
|
||||
worker.store.printStorePath(drvPath));
|
||||
buildLog = std::make_unique<BuildLog>(
|
||||
settings.logLines,
|
||||
std::make_unique<Activity>(
|
||||
*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), "", 1, 1}));
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
};
|
||||
|
||||
std::unique_ptr<Activity> actLock;
|
||||
DerivationBuilderUnique builder;
|
||||
Descriptor builderOut;
|
||||
|
||||
// Will continue here while waiting for a build user below
|
||||
@@ -666,7 +556,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
if (curBuilds >= settings.maxBuildJobs) {
|
||||
outputLocks.unlock();
|
||||
co_await waitForBuildSlot();
|
||||
co_return tryToBuild(std::move(inputPaths));
|
||||
co_return tryToBuild();
|
||||
}
|
||||
|
||||
if (!builder) {
|
||||
@@ -677,16 +567,10 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
struct DerivationBuildingGoalCallbacks : DerivationBuilderCallbacks
|
||||
{
|
||||
DerivationBuildingGoal & goal;
|
||||
std::function<void()> openLogFileFn;
|
||||
std::function<void()> closeLogFileFn;
|
||||
|
||||
DerivationBuildingGoalCallbacks(
|
||||
DerivationBuildingGoal & goal,
|
||||
std::function<void()> openLogFileFn,
|
||||
std::function<void()> closeLogFileFn)
|
||||
DerivationBuildingGoal & goal, std::unique_ptr<DerivationBuilder> & builder)
|
||||
: goal{goal}
|
||||
, openLogFileFn{std::move(openLogFileFn)}
|
||||
, closeLogFileFn{std::move(closeLogFileFn)}
|
||||
{
|
||||
}
|
||||
|
||||
@@ -694,17 +578,17 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
|
||||
void childTerminated() override
|
||||
{
|
||||
goal.worker.childTerminated(&goal, JobCategory::Build);
|
||||
goal.worker.childTerminated(&goal);
|
||||
}
|
||||
|
||||
void openLogFile() override
|
||||
std::filesystem::path openLogFile() override
|
||||
{
|
||||
openLogFileFn();
|
||||
return goal.openLogFile();
|
||||
}
|
||||
|
||||
void closeLogFile() override
|
||||
{
|
||||
closeLogFileFn();
|
||||
goal.closeLogFile();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -723,7 +607,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
worker.store.toStorePath(i.second.source.string()).first, closure);
|
||||
} catch (InvalidPath & e) {
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while processing sandbox path %s", PathFmt(i.second.source));
|
||||
e.addTrace({}, "while processing sandbox path '%s'", i.second.source);
|
||||
throw;
|
||||
}
|
||||
for (auto & i : closure) {
|
||||
@@ -754,16 +638,15 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
|
||||
/* If we have to wait and retry (see below), then `builder` will
|
||||
already be created, so we don't need to create it again. */
|
||||
builder = externalBuilder
|
||||
? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, openLogFile, closeLogFile),
|
||||
std::move(params));
|
||||
builder = externalBuilder ? makeExternalDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params),
|
||||
*externalBuilder)
|
||||
: makeDerivationBuilder(
|
||||
*localStoreP,
|
||||
std::make_unique<DerivationBuildingGoalCallbacks>(*this, builder),
|
||||
std::move(params));
|
||||
}
|
||||
|
||||
if (auto builderOutOpt = builder->startBuild()) {
|
||||
@@ -788,26 +671,16 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
|
||||
started();
|
||||
|
||||
uint64_t logSize = 0;
|
||||
|
||||
while (true) {
|
||||
auto event = co_await WaitForChildEvent{};
|
||||
if (auto * output = std::get_if<ChildOutput>(&event)) {
|
||||
if (output->fd == builder->builderOut.get()) {
|
||||
logSize += output->data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
builder->killChild();
|
||||
co_return doneFailureLogTooLong(*buildLog);
|
||||
}
|
||||
(*buildLog)(output->data);
|
||||
if (logFile->sink)
|
||||
(*logFile->sink)(output->data);
|
||||
}
|
||||
co_await processChildOutput(output->fd, output->data);
|
||||
} else if (std::get_if<ChildEOF>(&event)) {
|
||||
buildLog->flush();
|
||||
if (!currentLogLine.empty())
|
||||
flushLine();
|
||||
break;
|
||||
} else if (auto * timeout = std::get_if<TimedOut>(&event)) {
|
||||
builder->killChild();
|
||||
killChild();
|
||||
co_return doneFailure(std::move(*timeout));
|
||||
}
|
||||
}
|
||||
@@ -820,7 +693,7 @@ Goal::Co DerivationBuildingGoal::buildLocally(
|
||||
} catch (BuilderFailureError & e) {
|
||||
builder.reset();
|
||||
outputLocks.unlock();
|
||||
co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e), *buildLog));
|
||||
co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e)));
|
||||
} catch (BuildError & e) {
|
||||
builder.reset();
|
||||
outputLocks.unlock();
|
||||
@@ -941,7 +814,7 @@ static void runPostBuildHook(
|
||||
});
|
||||
}
|
||||
|
||||
BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e, BuildLog & buildLog)
|
||||
BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e)
|
||||
{
|
||||
auto msg =
|
||||
fmt("Cannot build '%s'.\n"
|
||||
@@ -951,7 +824,6 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
|
||||
|
||||
msg += showKnownOutputs(worker.store, *drv);
|
||||
|
||||
auto & logTail = buildLog.getTail();
|
||||
if (!logger->isVerbose() && !logTail.empty()) {
|
||||
msg += fmt("\nLast %d log lines:\n", logTail.size());
|
||||
for (auto & line : logTail) {
|
||||
@@ -974,7 +846,8 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
|
||||
return BuildError{e.status, msg};
|
||||
}
|
||||
|
||||
HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath> & drvOptions)
|
||||
HookReply DerivationBuildingGoal::tryBuildHook(
|
||||
const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return rpDecline;
|
||||
@@ -1030,8 +903,8 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
|
||||
else if (reply != "accept")
|
||||
throw Error("bad hook reply '%s'", reply);
|
||||
|
||||
} catch (SystemError & e) {
|
||||
if (e.is(std::errc::broken_pipe)) {
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == EPIPE) {
|
||||
printError("build hook died unexpectedly: %s", chomp(drainFD(worker.hook->fromHook.readSide.get())));
|
||||
worker.hook = 0;
|
||||
return rpDecline;
|
||||
@@ -1039,28 +912,70 @@ HookReply DerivationBuildingGoal::tryBuildHook(const DerivationOptions<StorePath
|
||||
throw;
|
||||
}
|
||||
|
||||
hook = std::move(worker.hook);
|
||||
|
||||
try {
|
||||
hook->machineName = readLine(hook->fromHook.readSide.get());
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while reading the machine name from the build hook");
|
||||
throw;
|
||||
}
|
||||
|
||||
CommonProto::WriteConn conn{hook->sink};
|
||||
|
||||
/* Tell the hook all the inputs that have to be copied to the
|
||||
remote system. */
|
||||
CommonProto::write(worker.store, conn, inputPaths);
|
||||
|
||||
/* Tell the hooks the missing outputs that have to be copied back
|
||||
from the remote system. */
|
||||
{
|
||||
StringSet missingOutputs;
|
||||
for (auto & [outputName, status] : initialOutputs) {
|
||||
// XXX: Does this include known CA outputs?
|
||||
if (buildMode != bmCheck && status.known && status.known->isValid())
|
||||
continue;
|
||||
missingOutputs.insert(outputName);
|
||||
}
|
||||
CommonProto::write(worker.store, conn, missingOutputs);
|
||||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
hook->toHook.writeSide.close();
|
||||
|
||||
/* Create the log file and pipe. */
|
||||
[[maybe_unused]] Path logFile = openLogFile();
|
||||
|
||||
std::set<MuxablePipePollState::CommChannel> fds;
|
||||
fds.insert(hook->fromHook.readSide.get());
|
||||
fds.insert(hook->builderOut.readSide.get());
|
||||
worker.childStarted(shared_from_this(), fds, false, false);
|
||||
|
||||
return rpAccept;
|
||||
#endif
|
||||
}
|
||||
|
||||
LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings & logSettings)
|
||||
Path DerivationBuildingGoal::openLogFile()
|
||||
{
|
||||
if (!logSettings.keepLog)
|
||||
return;
|
||||
logSize = 0;
|
||||
|
||||
auto baseName = std::string(baseNameOf(store.printStorePath(drvPath)));
|
||||
if (!settings.keepLog)
|
||||
return "";
|
||||
|
||||
auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
|
||||
|
||||
/* Create a log file. */
|
||||
Path logDir;
|
||||
if (auto localStore = dynamic_cast<LocalStore *>(&store))
|
||||
if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
|
||||
logDir = localStore->config->logDir;
|
||||
else
|
||||
logDir = logSettings.nixLogDir;
|
||||
logDir = settings.nixLogDir;
|
||||
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
|
||||
createDirs(dir);
|
||||
|
||||
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), logSettings.compressLog ? ".bz2" : "");
|
||||
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : "");
|
||||
|
||||
fd = toDescriptor(open(
|
||||
fdLogFile = toDescriptor(open(
|
||||
logFileName.c_str(),
|
||||
O_CREAT | O_WRONLY | O_TRUNC
|
||||
#ifndef _WIN32
|
||||
@@ -1068,37 +983,122 @@ LogFile::LogFile(Store & store, const StorePath & drvPath, const LogFileSettings
|
||||
#endif
|
||||
,
|
||||
0666));
|
||||
if (!fd)
|
||||
if (!fdLogFile)
|
||||
throw SysError("creating log file '%1%'", logFileName);
|
||||
|
||||
fileSink = std::make_shared<FdSink>(fd.get());
|
||||
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
||||
|
||||
if (logSettings.compressLog)
|
||||
sink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *fileSink));
|
||||
if (settings.compressLog)
|
||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
|
||||
else
|
||||
sink = fileSink;
|
||||
logSink = logFileSink;
|
||||
|
||||
return logFileName;
|
||||
}
|
||||
|
||||
LogFile::~LogFile()
|
||||
void DerivationBuildingGoal::closeLogFile()
|
||||
{
|
||||
try {
|
||||
auto sink2 = std::dynamic_pointer_cast<CompressionSink>(sink);
|
||||
if (sink2)
|
||||
sink2->finish();
|
||||
if (fileSink)
|
||||
fileSink->flush();
|
||||
} catch (...) {
|
||||
ignoreExceptionInDestructor();
|
||||
auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink);
|
||||
if (logSink2)
|
||||
logSink2->finish();
|
||||
if (logFileSink)
|
||||
logFileSink->flush();
|
||||
logSink = logFileSink = 0;
|
||||
fdLogFile.close();
|
||||
}
|
||||
|
||||
bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return false;
|
||||
#else
|
||||
return (hook && fd == hook->builderOut.readSide.get()) || (builder && fd == builder->builderOut.get());
|
||||
#endif
|
||||
}
|
||||
|
||||
Goal::Co DerivationBuildingGoal::processChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
if (isWrittenToLog) {
|
||||
logSize += data.size();
|
||||
if (settings.maxLogSize && logSize > settings.maxLogSize) {
|
||||
killChild();
|
||||
co_return doneFailure(BuildError(
|
||||
BuildResult::Failure::LogLimitExceeded,
|
||||
"%s killed after writing more than %d bytes of log output",
|
||||
getName(),
|
||||
settings.maxLogSize));
|
||||
}
|
||||
|
||||
for (auto c : data)
|
||||
if (c == '\r')
|
||||
currentLogLinePos = 0;
|
||||
else if (c == '\n')
|
||||
flushLine();
|
||||
else {
|
||||
if (currentLogLinePos >= currentLogLine.size())
|
||||
currentLogLine.resize(currentLogLinePos + 1);
|
||||
currentLogLine[currentLogLinePos++] = c;
|
||||
}
|
||||
|
||||
if (logSink)
|
||||
(*logSink)(data);
|
||||
}
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook && fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
auto json = parseJSONMessage(currentHookLine, "the derivation builder");
|
||||
if (json) {
|
||||
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true);
|
||||
// ensure that logs from a builder using `ssh-ng://` as protocol
|
||||
// are also available to `nix log`.
|
||||
if (s && !isWrittenToLog && logSink) {
|
||||
const auto type = (*json)["type"];
|
||||
const auto fields = (*json)["fields"];
|
||||
if (type == resBuildLogLine) {
|
||||
(*logSink)((fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n");
|
||||
} else if (type == resSetPhase && !fields.is_null()) {
|
||||
const auto phase = fields[0];
|
||||
if (!phase.is_null()) {
|
||||
// nixpkgs' stdenv produces lines in the log to signal
|
||||
// phase changes.
|
||||
// We want to get the same lines in case of remote builds.
|
||||
// The format is:
|
||||
// @nix { "action": "setPhase", "phase": "$curPhase" }
|
||||
const auto logLine = nlohmann::json::object({{"action", "setPhase"}, {"phase", phase}});
|
||||
(*logSink)(
|
||||
"@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)
|
||||
+ "\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
currentHookLine.clear();
|
||||
} else
|
||||
currentHookLine += c;
|
||||
}
|
||||
#endif
|
||||
co_return Return{};
|
||||
}
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneFailureLogTooLong(BuildLog & buildLog)
|
||||
void DerivationBuildingGoal::flushLine()
|
||||
{
|
||||
return doneFailure(BuildError(
|
||||
BuildResult::Failure::LogLimitExceeded,
|
||||
"%s killed after writing more than %d bytes of log output",
|
||||
getName(),
|
||||
settings.maxLogSize));
|
||||
if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false))
|
||||
;
|
||||
|
||||
else {
|
||||
logTail.push_back(currentLogLine);
|
||||
if (logTail.size() > settings.logLines)
|
||||
logTail.pop_front();
|
||||
|
||||
act->result(resBuildLogLine, currentLogLine);
|
||||
}
|
||||
|
||||
currentLogLine = "";
|
||||
currentLogLinePos = 0;
|
||||
}
|
||||
|
||||
std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPartialDerivationOutputMap()
|
||||
@@ -1186,6 +1186,11 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = std::move(builtOutputs),
|
||||
};
|
||||
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (status == BuildResult::Success::Built)
|
||||
@@ -1193,15 +1198,16 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status stat
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = std::move(builtOutputs),
|
||||
});
|
||||
return amDone(ecSuccess, std::nullopt);
|
||||
}
|
||||
|
||||
Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
|
||||
{
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
};
|
||||
|
||||
mcRunningBuilds.reset();
|
||||
|
||||
if (ex.status == BuildResult::Failure::TimedOut)
|
||||
@@ -1213,13 +1219,7 @@ Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
},
|
||||
std::move(ex));
|
||||
return amDone(ecFailed, {std::move(ex)});
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -465,6 +465,20 @@ UnkeyedRealisation DerivationGoal::assertPathValidity()
|
||||
|
||||
Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, UnkeyedRealisation builtOutput)
|
||||
{
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
};
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
|
||||
if (status == BuildResult::Success::Built)
|
||||
@@ -472,24 +486,16 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
});
|
||||
return amDone(ecSuccess, std::nullopt);
|
||||
}
|
||||
|
||||
Goal::Done DerivationGoal::doneFailure(BuildError ex)
|
||||
{
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
};
|
||||
|
||||
mcExpectedBuilds.reset();
|
||||
|
||||
if (ex.status == BuildResult::Failure::TimedOut)
|
||||
@@ -501,13 +507,7 @@ Goal::Done DerivationGoal::doneFailure(BuildError ex)
|
||||
|
||||
worker.updateProgress();
|
||||
|
||||
return Goal::doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = ex.status,
|
||||
.errorMsg = fmt("%s", Uncolored(ex.info().msg)),
|
||||
},
|
||||
std::move(ex));
|
||||
return amDone(ecFailed, {std::move(ex)});
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -98,13 +98,7 @@ Goal::Co DerivationTrampolineGoal::init()
|
||||
trace("outer load and build derivation");
|
||||
|
||||
if (nrFailed != 0) {
|
||||
co_return doneFailure(
|
||||
ecFailed,
|
||||
BuildResult::Failure{
|
||||
.status = BuildResult::Failure::DependencyFailed,
|
||||
.errorMsg = fmt("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)),
|
||||
},
|
||||
Error("failed to obtain derivation of '%s'", drvReq->to_string(worker.store)));
|
||||
co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
|
||||
}
|
||||
|
||||
StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
|
||||
|
||||
@@ -63,18 +63,12 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
|
||||
std::vector<KeyedBuildResult> results;
|
||||
results.reserve(state.size());
|
||||
|
||||
for (auto & [req, goalPtr] : state) {
|
||||
/* Goals that were never started or were cancelled have exitCode
|
||||
ecBusy and a default buildResult with empty errorMsg. Skip them
|
||||
to avoid reporting spurious failures with empty messages. */
|
||||
if (goalPtr->exitCode == Goal::ecBusy)
|
||||
continue;
|
||||
for (auto & [req, goalPtr] : state)
|
||||
results.emplace_back(
|
||||
KeyedBuildResult{
|
||||
goalPtr->buildResult,
|
||||
/* .path = */ req,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -183,19 +183,6 @@ Co Goal::await(Goals new_waitees)
|
||||
co_return Return{};
|
||||
}
|
||||
|
||||
Goal::Done Goal::doneSuccess(BuildResult::Success success)
|
||||
{
|
||||
buildResult.inner = std::move(success);
|
||||
return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
Goal::Done Goal::doneFailure(ExitCode result, BuildResult::Failure failure, std::optional<Error> ex)
|
||||
{
|
||||
assert(result == ecFailed || result == ecNoSubstituters);
|
||||
buildResult.inner = std::move(failure);
|
||||
return amDone(result, std::move(ex));
|
||||
}
|
||||
|
||||
Goal::Done Goal::amDone(ExitCode result, std::optional<Error> ex)
|
||||
{
|
||||
trace("done");
|
||||
|
||||
@@ -28,21 +28,20 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
|
||||
|
||||
Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status)
|
||||
{
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
});
|
||||
buildResult.inner = BuildResult::Success{
|
||||
.status = status,
|
||||
};
|
||||
return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg)
|
||||
{
|
||||
debug(errorMsg);
|
||||
return Goal::doneFailure(
|
||||
result,
|
||||
BuildResult::Failure{
|
||||
.status = status,
|
||||
.errorMsg = std::move(errorMsg),
|
||||
});
|
||||
buildResult.inner = BuildResult::Failure{
|
||||
.status = status,
|
||||
.errorMsg = std::move(errorMsg),
|
||||
};
|
||||
return amDone(result);
|
||||
}
|
||||
|
||||
Goal::Co PathSubstitutionGoal::init()
|
||||
@@ -325,7 +324,7 @@ void PathSubstitutionGoal::cleanup()
|
||||
if (thr.joinable()) {
|
||||
// FIXME: signal worker thread to quit.
|
||||
thr.join();
|
||||
worker.childTerminated(this, JobCategory::Substitution);
|
||||
worker.childTerminated(this);
|
||||
}
|
||||
|
||||
outPipe.close();
|
||||
|
||||
@@ -162,9 +162,7 @@ template<typename G>
|
||||
static bool
|
||||
removeGoal(std::shared_ptr<G> goal, typename DerivedPathMap<std::map<OutputsSpec, std::weak_ptr<G>>>::ChildNode & node)
|
||||
{
|
||||
bool valueKeep = removeGoal(goal, node.value);
|
||||
bool childMapKeep = removeGoal(goal, node.childMap);
|
||||
return valueKeep || childMapKeep;
|
||||
return removeGoal(goal, node.value) || removeGoal(goal, node.childMap);
|
||||
}
|
||||
|
||||
void Worker::removeGoal(GoalPtr goal)
|
||||
@@ -247,18 +245,13 @@ void Worker::childStarted(
|
||||
}
|
||||
|
||||
void Worker::childTerminated(Goal * goal, bool wakeSleepers)
|
||||
{
|
||||
childTerminated(goal, goal->jobCategory(), wakeSleepers);
|
||||
}
|
||||
|
||||
void Worker::childTerminated(Goal * goal, JobCategory jobCategory, bool wakeSleepers)
|
||||
{
|
||||
auto i = std::find_if(children.begin(), children.end(), [&](const Child & child) { return child.goal2 == goal; });
|
||||
if (i == children.end())
|
||||
return;
|
||||
|
||||
if (i->inBuildSlot) {
|
||||
switch (jobCategory) {
|
||||
switch (goal->jobCategory()) {
|
||||
case JobCategory::Substitution:
|
||||
assert(nrSubstitutions > 0);
|
||||
nrSubstitutions--;
|
||||
@@ -408,12 +401,8 @@ void Worker::waitForInput()
|
||||
is a build timeout, then wait for input until the first
|
||||
deadline for any child. */
|
||||
auto nearest = steady_time_point::max(); // nearest deadline
|
||||
|
||||
auto localStore = dynamic_cast<LocalStore *>(&store);
|
||||
if (localStore && localStore->config->getGCSettings().minFree.get() != 0)
|
||||
// If we have a local store (and thus are capable of automatically collecting garbage) and configured to do so,
|
||||
// periodically wake up to see if we need to run the garbage collector. (See the `autoGC` call site above in
|
||||
// this file, also gated on having a local store. when we wake up, we intended to reach that call site.)
|
||||
if (settings.minFree.get() != 0)
|
||||
// Periodicallty wake up to see if we need to run the garbage collector.
|
||||
nearest = before + std::chrono::seconds(10);
|
||||
for (auto & i : children) {
|
||||
if (!i.respectTimeouts)
|
||||
|
||||
@@ -33,8 +33,8 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
|
||||
|
||||
try {
|
||||
srcFiles = DirectoryIterator{srcDir};
|
||||
} catch (SystemError & e) {
|
||||
if (e.is(std::errc::not_a_directory)) {
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == ENOTDIR) {
|
||||
warn("not including '%s' in the user environment because it's not a directory", srcDir);
|
||||
return;
|
||||
}
|
||||
@@ -54,8 +54,8 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
|
||||
try {
|
||||
if (stat(srcFile.c_str(), &srcSt) == -1)
|
||||
throw SysError("getting status of '%1%'", srcFile);
|
||||
} catch (SystemError & e) {
|
||||
if (e.is(std::errc::no_such_file_or_directory) || e.is(std::errc::not_a_directory)) {
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
|
||||
warn("skipping dangling symlink '%s'", dstFile);
|
||||
continue;
|
||||
}
|
||||
@@ -141,8 +141,8 @@ void buildProfile(const Path & out, Packages && pkgs)
|
||||
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
|
||||
if (!done.count(p))
|
||||
postponed.insert(p);
|
||||
} catch (SystemError & e) {
|
||||
if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::not_a_directory))
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo != ENOENT && e.errNo != ENOTDIR)
|
||||
throw;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
#include "nix/store/globals.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/util/compression.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -66,7 +65,8 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
|
||||
|
||||
auto executable = ctx.drv.env.find("executable");
|
||||
if (executable != ctx.drv.env.end() && executable->second == "1") {
|
||||
chmod(storePath, 0755);
|
||||
if (chmod(storePath.c_str(), 0755) == -1)
|
||||
throw SysError("making '%1%' executable", storePath);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include "nix/store/common-protocol-impl.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/store/derivations.hh"
|
||||
#include "nix/util/signature/local-keys.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@@ -100,15 +99,4 @@ void CommonProto::Serialise<std::optional<ContentAddress>>::write(
|
||||
conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
|
||||
}
|
||||
|
||||
Signature CommonProto::Serialise<Signature>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return Signature::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<Signature>::write(
|
||||
const StoreDirConfig & store, CommonProto::WriteConn conn, const Signature & sig)
|
||||
{
|
||||
conn.to << sig.to_string();
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
+31
-3
@@ -339,6 +339,17 @@ static void performOp(
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::HasSubstitutes: {
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
StorePathSet paths; // FIXME
|
||||
paths.insert(path);
|
||||
auto res = store->querySubstitutablePaths(paths);
|
||||
logger->stopWork();
|
||||
conn.to << (res.count(path) != 0);
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::QuerySubstitutablePaths: {
|
||||
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
@@ -348,13 +359,26 @@ static void performOp(
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::QueryPathHash: {
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
auto hash = store->queryPathInfo(path)->narHash;
|
||||
logger->stopWork();
|
||||
conn.to << hash.to_string(HashFormat::Base16, false);
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::QueryReferences:
|
||||
case WorkerProto::Op::QueryReferrers:
|
||||
case WorkerProto::Op::QueryValidDerivers:
|
||||
case WorkerProto::Op::QueryDerivationOutputs: {
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
StorePathSet paths;
|
||||
if (op == WorkerProto::Op::QueryReferrers)
|
||||
if (op == WorkerProto::Op::QueryReferences)
|
||||
for (auto & i : store->queryPathInfo(path)->references)
|
||||
paths.insert(i);
|
||||
else if (op == WorkerProto::Op::QueryReferrers)
|
||||
store->queryReferrers(path, paths);
|
||||
else if (op == WorkerProto::Op::QueryValidDerivers)
|
||||
paths = store->queryValidDerivers(path);
|
||||
@@ -861,7 +885,7 @@ static void performOp(
|
||||
|
||||
case WorkerProto::Op::AddSignatures: {
|
||||
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
|
||||
auto sigs = WorkerProto::Serialise<std::set<Signature>>::read(*store, rconn);
|
||||
StringSet sigs = readStrings<StringSet>(conn.from);
|
||||
logger->startWork();
|
||||
store->addSignatures(path, sigs);
|
||||
logger->stopWork();
|
||||
@@ -886,7 +910,7 @@ static void performOp(
|
||||
info.deriver = std::move(deriver);
|
||||
info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
conn.from >> info.registrationTime >> info.narSize >> info.ultimate;
|
||||
info.sigs = WorkerProto::Serialise<std::set<Signature>>::read(*store, rconn);
|
||||
info.sigs = readStrings<StringSet>(conn.from);
|
||||
info.ca = ContentAddress::parseOpt(readString(conn.from));
|
||||
conn.from >> repair >> dontCheckSigs;
|
||||
if (!trusted && dontCheckSigs)
|
||||
@@ -988,6 +1012,10 @@ static void performOp(
|
||||
break;
|
||||
}
|
||||
|
||||
case WorkerProto::Op::QueryFailedPaths:
|
||||
case WorkerProto::Op::ClearFailedPaths:
|
||||
throw Error("Removed operation %1%", op);
|
||||
|
||||
default:
|
||||
throw Error("invalid operation %1%", op);
|
||||
}
|
||||
|
||||
@@ -21,53 +21,76 @@ static std::optional<std::string>
|
||||
getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
|
||||
{
|
||||
if (parsed) {
|
||||
if (auto * i = get(parsed->structuredAttrs, name))
|
||||
try {
|
||||
return getString(*i);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing attribute \"%s\"", name);
|
||||
throw;
|
||||
}
|
||||
auto i = parsed->structuredAttrs.find(name);
|
||||
if (i == parsed->structuredAttrs.end())
|
||||
return {};
|
||||
else {
|
||||
if (!i->second.is_string())
|
||||
throw Error("attribute '%s' of must be a string", name);
|
||||
return i->second.get<std::string>();
|
||||
}
|
||||
} else {
|
||||
if (auto * i = get(env, name))
|
||||
return *i;
|
||||
auto i = env.find(name);
|
||||
if (i == env.end())
|
||||
return {};
|
||||
else
|
||||
return i->second;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name, bool def)
|
||||
{
|
||||
if (parsed) {
|
||||
if (auto * i = get(parsed->structuredAttrs, name))
|
||||
try {
|
||||
return getBoolean(*i);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing attribute \"%s\"", name);
|
||||
throw;
|
||||
}
|
||||
auto i = parsed->structuredAttrs.find(name);
|
||||
if (i == parsed->structuredAttrs.end())
|
||||
return def;
|
||||
else {
|
||||
if (!i->second.is_boolean())
|
||||
throw Error("attribute '%s' must be a Boolean", name);
|
||||
return i->second.get<bool>();
|
||||
}
|
||||
} else {
|
||||
if (auto * i = get(env, name))
|
||||
return *i == "1";
|
||||
auto i = env.find(name);
|
||||
if (i == env.end())
|
||||
return def;
|
||||
else
|
||||
return i->second == "1";
|
||||
}
|
||||
}
|
||||
|
||||
static std::optional<Strings>
|
||||
getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
|
||||
{
|
||||
if (parsed) {
|
||||
auto i = parsed->structuredAttrs.find(name);
|
||||
if (i == parsed->structuredAttrs.end())
|
||||
return {};
|
||||
else {
|
||||
if (!i->second.is_array())
|
||||
throw Error("attribute '%s' must be a list of strings", name);
|
||||
auto & a = getArray(i->second);
|
||||
Strings res;
|
||||
for (auto j = a.begin(); j != a.end(); ++j) {
|
||||
if (!j->is_string())
|
||||
throw Error("attribute '%s' must be a list of strings", name);
|
||||
res.push_back(j->get<std::string>());
|
||||
}
|
||||
return res;
|
||||
}
|
||||
} else {
|
||||
auto i = env.find(name);
|
||||
if (i == env.end())
|
||||
return {};
|
||||
else
|
||||
return tokenizeString<Strings>(i->second);
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
static std::optional<StringSet>
|
||||
getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
|
||||
{
|
||||
if (parsed) {
|
||||
if (auto * i = get(parsed->structuredAttrs, name))
|
||||
try {
|
||||
return getStringSet(*i);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing attribute \"%s\"", name);
|
||||
throw;
|
||||
}
|
||||
} else {
|
||||
if (auto * i = get(env, name))
|
||||
return tokenizeString<StringSet>(*i);
|
||||
}
|
||||
return {};
|
||||
auto ss = getStringsAttr(env, parsed, name);
|
||||
return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
|
||||
}
|
||||
|
||||
template<typename Inputs>
|
||||
@@ -210,21 +233,26 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
|
||||
std::map<std::string, OutputChecks<SingleDerivedPath>> res;
|
||||
if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
|
||||
for (auto & [outputName, output_] : getObject(*outputChecks)) {
|
||||
OutputChecks<SingleDerivedPath> checks;
|
||||
|
||||
auto & output = getObject(output_);
|
||||
|
||||
if (auto maxSize = get(output, "maxSize"))
|
||||
checks.maxSize = maxSize->get<uint64_t>();
|
||||
|
||||
if (auto maxClosureSize = get(output, "maxClosureSize"))
|
||||
checks.maxClosureSize = maxClosureSize->get<uint64_t>();
|
||||
|
||||
auto get_ =
|
||||
[&](const std::string & name) -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
|
||||
if (auto * i = get(output, name)) {
|
||||
try {
|
||||
std::set<DrvRef<SingleDerivedPath>> res;
|
||||
for (auto & s : getStringList(*i))
|
||||
res.insert(parseRef(s));
|
||||
return res;
|
||||
} catch (Error & e) {
|
||||
e.addTrace(
|
||||
{}, "while parsing attribute 'outputChecks.\"%s\".%s'", outputName, name);
|
||||
throw;
|
||||
if (auto i = get(output, name)) {
|
||||
std::set<DrvRef<SingleDerivedPath>> res;
|
||||
for (auto j = i->begin(); j != i->end(); ++j) {
|
||||
if (!j->is_string())
|
||||
throw Error("attribute '%s' must be a list of strings", name);
|
||||
res.insert(parseRef(j->get<std::string>()));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
return {};
|
||||
};
|
||||
@@ -232,8 +260,18 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
|
||||
res.insert_or_assign(
|
||||
outputName,
|
||||
OutputChecks<SingleDerivedPath>{
|
||||
.maxSize = ptrToOwned<uint64_t>(get(output, "maxSize")),
|
||||
.maxClosureSize = ptrToOwned<uint64_t>(get(output, "maxClosureSize")),
|
||||
.maxSize = [&]() -> std::optional<uint64_t> {
|
||||
if (auto maxSize = get(output, "maxSize"))
|
||||
return maxSize->get<uint64_t>();
|
||||
else
|
||||
return std::nullopt;
|
||||
}(),
|
||||
.maxClosureSize = [&]() -> std::optional<uint64_t> {
|
||||
if (auto maxClosureSize = get(output, "maxClosureSize"))
|
||||
return maxClosureSize->get<uint64_t>();
|
||||
else
|
||||
return std::nullopt;
|
||||
}(),
|
||||
.allowedReferences = get_("allowedReferences"),
|
||||
.disallowedReferences =
|
||||
get_("disallowedReferences").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
|
||||
@@ -269,13 +307,13 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
|
||||
std::map<std::string, bool> res;
|
||||
|
||||
if (parsed) {
|
||||
if (auto * udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) {
|
||||
try {
|
||||
for (auto & [outputName, output] : getObject(*udr))
|
||||
res.insert_or_assign(outputName, getBoolean(output));
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing attribute 'unsafeDiscardReferences'");
|
||||
throw;
|
||||
auto & structuredAttrs = parsed->structuredAttrs;
|
||||
|
||||
if (auto * udr = get(structuredAttrs, "unsafeDiscardReferences")) {
|
||||
for (auto & [outputName, output] : getObject(*udr)) {
|
||||
if (!output.is_boolean())
|
||||
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName);
|
||||
res.insert_or_assign(outputName, output.get<bool>());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -302,7 +340,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
|
||||
std::map<std::string, std::set<SingleDerivedPath>> ret;
|
||||
|
||||
if (parsed) {
|
||||
auto * e = get(parsed->structuredAttrs, "exportReferencesGraph");
|
||||
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
|
||||
if (!e)
|
||||
return ret;
|
||||
if (!e->is_object()) {
|
||||
@@ -557,8 +595,8 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
|
||||
.outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> {
|
||||
auto outputChecks = getObject(valueAt(json, "outputChecks"));
|
||||
|
||||
auto forAllOutputsOpt = get(outputChecks, "forAllOutputs");
|
||||
auto perOutputOpt = get(outputChecks, "perOutput");
|
||||
auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs");
|
||||
auto perOutputOpt = optionalValueAt(outputChecks, "perOutput");
|
||||
|
||||
if (forAllOutputsOpt && !perOutputOpt) {
|
||||
return static_cast<OutputChecks<SingleDerivedPath>>(*forAllOutputsOpt);
|
||||
|
||||
@@ -80,6 +80,12 @@ public:
|
||||
subdirs.emplace(baseName, std::move(accessor));
|
||||
}
|
||||
|
||||
std::string readFile(const CanonPath & path) override
|
||||
{
|
||||
return callWithAccessorForPath(
|
||||
path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readFile(path); });
|
||||
}
|
||||
|
||||
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
|
||||
{
|
||||
return callWithAccessorForPath(path, [&](SourceAccessor & accessor, const CanonPath & path) {
|
||||
|
||||
@@ -69,7 +69,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
curlFileTransfer & fileTransfer;
|
||||
FileTransferRequest request;
|
||||
FileTransferResult result;
|
||||
std::unique_ptr<Activity> _act;
|
||||
Activity act;
|
||||
bool done = false; // whether either the success or failure function has been called
|
||||
Callback<FileTransferResult> callback;
|
||||
CURL * req = 0;
|
||||
@@ -124,6 +124,12 @@ struct curlFileTransfer : public FileTransfer
|
||||
Callback<FileTransferResult> && callback)
|
||||
: fileTransfer(fileTransfer)
|
||||
, request(request)
|
||||
, act(*logger,
|
||||
lvlTalkative,
|
||||
actFileTransfer,
|
||||
fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri),
|
||||
{request.uri.to_string()},
|
||||
request.parentAct)
|
||||
, callback(std::move(callback))
|
||||
, finalSink([this](std::string_view data) {
|
||||
if (errorSink) {
|
||||
@@ -151,15 +157,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
result.urls.push_back(request.uri.to_string());
|
||||
|
||||
/* Don't set Accept-Encoding for S3 requests that use AWS SigV4 signing.
|
||||
curl's SigV4 implementation signs all headers including Accept-Encoding,
|
||||
but some S3-compatible services (like GCS) modify this header in transit,
|
||||
causing signature verification to fail.
|
||||
See https://github.com/NixOS/nix/issues/15019 */
|
||||
#if NIX_WITH_AWS_AUTH
|
||||
if (!request.awsSigV4Provider)
|
||||
#endif
|
||||
appendHeaders("Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
appendHeaders("Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
|
||||
if (!request.expectedETag.empty())
|
||||
appendHeaders("If-None-Match: " + request.expectedETag);
|
||||
if (!request.mimeType.empty())
|
||||
@@ -327,29 +325,9 @@ struct curlFileTransfer : public FileTransfer
|
||||
return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily start an `Activity`. We don't do this in the `TransferItem` constructor to avoid showing downloads
|
||||
* that are only enqueued but not actually started.
|
||||
*/
|
||||
Activity & act()
|
||||
{
|
||||
if (!_act) {
|
||||
_act = std::make_unique<Activity>(
|
||||
*logger,
|
||||
lvlTalkative,
|
||||
actFileTransfer,
|
||||
fmt("%s '%s'", request.verb(/*continuous=*/true), request.uri),
|
||||
Logger::Fields{request.uri.to_string()},
|
||||
request.parentAct);
|
||||
// Reset the start time to when we actually started the download.
|
||||
startTime = std::chrono::steady_clock::now();
|
||||
}
|
||||
return *_act;
|
||||
}
|
||||
|
||||
int progressCallback(curl_off_t dltotal, curl_off_t dlnow) noexcept
|
||||
try {
|
||||
act().progress(dlnow, dltotal);
|
||||
act.progress(dlnow, dltotal);
|
||||
return getInterrupted();
|
||||
} catch (nix::Interrupted &) {
|
||||
assert(getInterrupted());
|
||||
@@ -426,15 +404,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
return ((TransferItem *) clientp)->seekCallback(offset, origin);
|
||||
}
|
||||
|
||||
static int resolverCallbackWrapper(void *, void *, void * clientp) noexcept
|
||||
try {
|
||||
// Create the `Activity` associated with this download.
|
||||
((TransferItem *) clientp)->act();
|
||||
return 0;
|
||||
} catch (...) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
void unpause()
|
||||
{
|
||||
/* Unpausing an already unpaused transfer is a no-op. */
|
||||
@@ -492,11 +461,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
|
||||
|
||||
if (request.data) {
|
||||
// Restart the source to ensure it's at the beginning.
|
||||
// This is necessary for retries, where the source was
|
||||
// already consumed by a previous attempt.
|
||||
request.data->source->restart();
|
||||
|
||||
if (request.method == HttpMethod::Post) {
|
||||
curl_easy_setopt(req, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
|
||||
@@ -531,21 +495,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (writtenToSink)
|
||||
curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
|
||||
|
||||
/* Note that the underlying strings get copied by libcurl, so the path -> string conversion is ok:
|
||||
> The application does not have to keep the string around after setting this option.
|
||||
https://curl.se/libcurl/c/CURLOPT_SSLKEY.html
|
||||
https://curl.se/libcurl/c/CURLOPT_SSLCERT.html */
|
||||
|
||||
if (request.tlsCert) {
|
||||
curl_easy_setopt(req, CURLOPT_SSLCERTTYPE, "PEM");
|
||||
curl_easy_setopt(req, CURLOPT_SSLCERT, request.tlsCert->string().c_str());
|
||||
}
|
||||
|
||||
if (request.tlsKey) {
|
||||
curl_easy_setopt(req, CURLOPT_SSLKEYTYPE, "PEM");
|
||||
curl_easy_setopt(req, CURLOPT_SSLKEY, request.tlsKey->string().c_str());
|
||||
}
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf);
|
||||
errbuf[0] = 0;
|
||||
|
||||
@@ -567,11 +516,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
}
|
||||
#endif
|
||||
|
||||
// This seems to be the earliest libcurl callback that signals that the download is happening, so we can
|
||||
// call act().
|
||||
curl_easy_setopt(req, CURLOPT_RESOLVER_START_FUNCTION, resolverCallbackWrapper);
|
||||
curl_easy_setopt(req, CURLOPT_RESOLVER_START_DATA, this);
|
||||
|
||||
result.data.clear();
|
||||
result.bodySize = 0;
|
||||
}
|
||||
@@ -620,7 +564,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (httpStatus == 304 && result.etag == "")
|
||||
result.etag = request.expectedETag;
|
||||
|
||||
act().progress(result.bodySize, result.bodySize);
|
||||
act.progress(result.bodySize, result.bodySize);
|
||||
done = true;
|
||||
callback(std::move(result));
|
||||
}
|
||||
@@ -771,8 +715,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
std::thread workerThread;
|
||||
|
||||
const size_t maxQueueSize = fileTransferSettings.httpConnections.get() * 5;
|
||||
|
||||
curlFileTransfer()
|
||||
: mt19937(rd())
|
||||
{
|
||||
@@ -878,13 +820,6 @@ struct curlFileTransfer : public FileTransfer
|
||||
{
|
||||
auto state(state_.lock());
|
||||
while (!state->incoming.empty()) {
|
||||
/* Limit the number of active curl handles, since curl doesn't scale well. */
|
||||
if (items.size() + incoming.size() >= maxQueueSize) {
|
||||
auto t = now + std::chrono::milliseconds(100);
|
||||
if (nextWakeup == std::chrono::steady_clock::time_point() || t < nextWakeup)
|
||||
nextWakeup = t;
|
||||
break;
|
||||
}
|
||||
auto item = state->incoming.top();
|
||||
if (item->embargo <= now) {
|
||||
incoming.push_back(item);
|
||||
|
||||
+24
-29
@@ -115,10 +115,10 @@ restart:
|
||||
*fdRootsSocket = createUnixDomainSocket();
|
||||
try {
|
||||
nix::connect(toSocket(fdRootsSocket->get()), socketPath);
|
||||
} catch (SystemError & e) {
|
||||
} catch (SysError & e) {
|
||||
/* The garbage collector may have exited or not
|
||||
created the socket yet, so we need to restart. */
|
||||
if (e.is(std::errc::connection_refused) || e.is(std::errc::no_such_file_or_directory)) {
|
||||
if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) {
|
||||
debug("GC socket connection refused: %s", e.msg());
|
||||
fdRootsSocket->close();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
@@ -135,10 +135,10 @@ restart:
|
||||
readFull(fdRootsSocket->get(), &c, 1);
|
||||
assert(c == '1');
|
||||
debug("got ack for GC root '%s'", printStorePath(path));
|
||||
} catch (SystemError & e) {
|
||||
} catch (SysError & e) {
|
||||
/* The garbage collector may have exited, so we need to
|
||||
restart. */
|
||||
if (e.is(std::errc::broken_pipe) || e.is(std::errc::connection_reset)) {
|
||||
if (e.errNo == EPIPE || e.errNo == ECONNRESET) {
|
||||
debug("GC socket disconnected");
|
||||
fdRootsSocket->close();
|
||||
goto restart;
|
||||
@@ -280,10 +280,9 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
|
||||
throw;
|
||||
}
|
||||
|
||||
catch (SystemError & e) {
|
||||
catch (SysError & e) {
|
||||
/* We only ignore permanent failures. */
|
||||
if (e.is(std::errc::permission_denied) || e.is(std::errc::no_such_file_or_directory)
|
||||
|| e.is(std::errc::not_a_directory))
|
||||
if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
|
||||
printInfo("cannot read potential root '%1%'", path);
|
||||
else
|
||||
throw;
|
||||
@@ -348,8 +347,8 @@ static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & r
|
||||
{
|
||||
try {
|
||||
roots[readFile(path)].emplace(path.string());
|
||||
} catch (SystemError & e) {
|
||||
if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::permission_denied))
|
||||
} catch (SysError & e) {
|
||||
if (e.errNo != ENOENT && e.errNo != EACCES)
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -465,11 +464,9 @@ struct GCLimitReached
|
||||
|
||||
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
{
|
||||
const auto & gcSettings = config->getGCSettings();
|
||||
|
||||
bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
|
||||
bool keepOutputs = gcSettings.keepOutputs;
|
||||
bool keepDerivations = gcSettings.keepDerivations;
|
||||
bool gcKeepOutputs = settings.gcKeepOutputs;
|
||||
bool gcKeepDerivations = settings.gcKeepDerivations;
|
||||
|
||||
boost::unordered_flat_set<StorePath, std::hash<StorePath>> roots, dead, alive;
|
||||
|
||||
@@ -493,8 +490,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
(the garbage collector will recurse into deleting the outputs
|
||||
or derivers, respectively). So disable them. */
|
||||
if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
|
||||
keepOutputs = false;
|
||||
keepDerivations = false;
|
||||
gcKeepOutputs = false;
|
||||
gcKeepDerivations = false;
|
||||
}
|
||||
|
||||
if (shouldDelete)
|
||||
@@ -651,7 +648,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
|
||||
/* Helper function that deletes a path from the store and throws
|
||||
GCLimitReached if we've deleted enough garbage. */
|
||||
auto deleteFromStore = [&](std::string_view baseName, bool isKnownPath) {
|
||||
auto deleteFromStore = [&](std::string_view baseName) {
|
||||
Path path = storeDir + "/" + std::string(baseName);
|
||||
Path realPath = config->realStoreDir + "/" + std::string(baseName);
|
||||
|
||||
@@ -671,7 +668,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
results.paths.insert(path);
|
||||
|
||||
uint64_t bytesFreed;
|
||||
deleteStorePath(realPath, bytesFreed, isKnownPath);
|
||||
deleteStorePath(realPath, bytesFreed);
|
||||
|
||||
results.bytesFreed += bytesFreed;
|
||||
|
||||
@@ -730,8 +727,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
*path,
|
||||
closure,
|
||||
/* flipDirection */ false,
|
||||
keepOutputs,
|
||||
keepDerivations);
|
||||
gcKeepOutputs,
|
||||
gcKeepDerivations);
|
||||
for (auto & p : closure)
|
||||
alive.insert(p);
|
||||
} catch (InvalidPath &) {
|
||||
@@ -772,7 +769,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
|
||||
/* If keep-derivations is set and this is a
|
||||
derivation, then visit the derivation outputs. */
|
||||
if (keepDerivations && path->isDerivation()) {
|
||||
if (gcKeepDerivations && path->isDerivation()) {
|
||||
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
|
||||
if (maybeOutPath && isValidPath(*maybeOutPath)
|
||||
&& queryPathInfo(*maybeOutPath)->deriver == *path)
|
||||
@@ -780,7 +777,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
}
|
||||
|
||||
/* If keep-outputs is set, then visit the derivers. */
|
||||
if (keepOutputs) {
|
||||
if (gcKeepOutputs) {
|
||||
auto derivers = queryValidDerivers(*path);
|
||||
for (auto & i : derivers)
|
||||
enqueue(i);
|
||||
@@ -793,7 +790,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
if (shouldDelete) {
|
||||
try {
|
||||
invalidatePathChecked(path);
|
||||
deleteFromStore(path.to_string(), true);
|
||||
deleteFromStore(path.to_string());
|
||||
referrersCache.erase(path);
|
||||
} catch (PathInUse & e) {
|
||||
// If we end up here, it's likely a new occurrence
|
||||
@@ -844,7 +841,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
|
||||
deleteReferrersClosure(*storePath);
|
||||
else
|
||||
deleteFromStore(name, false);
|
||||
deleteFromStore(name);
|
||||
}
|
||||
} catch (GCLimitReached & e) {
|
||||
}
|
||||
@@ -922,8 +919,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
||||
void LocalStore::autoGC(bool sync)
|
||||
{
|
||||
#if HAVE_STATVFS
|
||||
const auto & gcSettings = config->getGCSettings();
|
||||
|
||||
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE");
|
||||
|
||||
auto getAvail = [this]() -> uint64_t {
|
||||
@@ -950,14 +945,14 @@ void LocalStore::autoGC(bool sync)
|
||||
|
||||
auto now = std::chrono::steady_clock::now();
|
||||
|
||||
if (now < state->lastGCCheck + std::chrono::seconds(gcSettings.minFreeCheckInterval))
|
||||
if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval))
|
||||
return;
|
||||
|
||||
auto avail = getAvail();
|
||||
|
||||
state->lastGCCheck = now;
|
||||
|
||||
if (avail >= gcSettings.minFree || avail >= gcSettings.maxFree)
|
||||
if (avail >= settings.minFree || avail >= settings.maxFree)
|
||||
return;
|
||||
|
||||
if (avail > state->availAfterGC * 0.97)
|
||||
@@ -968,7 +963,7 @@ void LocalStore::autoGC(bool sync)
|
||||
std::promise<void> promise;
|
||||
future = state->gcFuture = promise.get_future().share();
|
||||
|
||||
std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable {
|
||||
std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable {
|
||||
try {
|
||||
|
||||
/* Wake up any threads waiting for the auto-GC to finish. */
|
||||
@@ -980,7 +975,7 @@ void LocalStore::autoGC(bool sync)
|
||||
});
|
||||
|
||||
GCOptions options;
|
||||
options.maxFreed = gcSettings.maxFree - avail;
|
||||
options.maxFreed = settings.maxFree - avail;
|
||||
|
||||
printInfo("running auto-GC to free %d bytes", options.maxFreed);
|
||||
|
||||
|
||||
+5
-32
@@ -35,10 +35,6 @@
|
||||
# include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
# include "nix/util/windows-known-folders.hh"
|
||||
#endif
|
||||
|
||||
#include "store-config-private.hh"
|
||||
|
||||
namespace nix {
|
||||
@@ -50,46 +46,23 @@ namespace nix {
|
||||
must be deleted and recreated on startup.) */
|
||||
#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
|
||||
|
||||
/**
|
||||
* Helper to resolve the NIX_CONF_DIR at runtime on Windows.
|
||||
* On Windows, NIX_CONF_DIR is not defined at compile time, so we determine
|
||||
* the path at runtime using the Windows known folders API (FOLDERID_ProgramData).
|
||||
* This allows Nix to work correctly regardless of which drive Windows is installed on.
|
||||
*/
|
||||
static std::filesystem::path resolveNixConfDir()
|
||||
{
|
||||
#ifdef _WIN32
|
||||
# ifdef NIX_CONF_DIR
|
||||
// On Windows, NIX_CONF_DIR should not be defined at compile time
|
||||
# error "NIX_CONF_DIR should not be defined on Windows"
|
||||
# endif
|
||||
return windows::known_folders::getProgramData() / "nix";
|
||||
#else
|
||||
return NIX_CONF_DIR;
|
||||
#endif
|
||||
}
|
||||
|
||||
LogFileSettings::LogFileSettings()
|
||||
: nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
|
||||
{
|
||||
}
|
||||
|
||||
Settings settings;
|
||||
|
||||
static GlobalConfig::Register rSettings(&settings);
|
||||
|
||||
Settings::Settings()
|
||||
: nixStore(
|
||||
: nixPrefix(NIX_PREFIX)
|
||||
, nixStore(
|
||||
#ifndef _WIN32
|
||||
// On Windows `/nix/store` is not a canonical path, but we dont'
|
||||
// want to deal with that yet.
|
||||
canonPath
|
||||
#endif
|
||||
(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR))))
|
||||
, nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR)))
|
||||
, nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
|
||||
, nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
|
||||
, nixConfDir(canonPath(getEnvOsNonEmpty(OS_STR("NIX_CONF_DIR"))
|
||||
.transform([](auto && s) { return std::filesystem::path(s); })
|
||||
.value_or(resolveNixConfDir())))
|
||||
, nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
|
||||
, nixUserConfFiles(getUserConfigFiles())
|
||||
, nixDaemonSocketFile(
|
||||
canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
|
||||
|
||||
@@ -78,13 +78,13 @@ void HttpBinaryCacheStore::init()
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<CompressionAlgo> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
|
||||
std::optional<std::string> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
|
||||
{
|
||||
if (hasSuffix(path, ".narinfo") && config->narinfoCompression.get())
|
||||
if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty())
|
||||
return config->narinfoCompression;
|
||||
else if (hasSuffix(path, ".ls") && config->lsCompression.get())
|
||||
else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty())
|
||||
return config->lsCompression;
|
||||
else if (hasPrefix(path, "log/") && config->logCompression.get())
|
||||
else if (hasPrefix(path, "log/") && !config->logCompression.get().empty())
|
||||
return config->logCompression;
|
||||
else
|
||||
return std::nullopt;
|
||||
@@ -160,9 +160,7 @@ void HttpBinaryCacheStore::upsertFile(
|
||||
try {
|
||||
if (auto compressionMethod = getCompressionMethod(path)) {
|
||||
CompressedSource compressed(source, *compressionMethod);
|
||||
/* TODO: Validate that this is a valid content encoding. We probably shouldn't set non-standard values here.
|
||||
*/
|
||||
Headers headers = {{"Content-Encoding", showCompressionAlgo(*compressionMethod)}};
|
||||
Headers headers = {{"Content-Encoding", *compressionMethod}};
|
||||
upload(path, compressed, compressed.size(), mimeType, std::move(headers));
|
||||
} else {
|
||||
upload(path, source, sizeHint, mimeType, std::nullopt);
|
||||
@@ -195,23 +193,7 @@ FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
|
||||
result.query = config->cacheUri.query;
|
||||
}
|
||||
|
||||
FileTransferRequest request(result);
|
||||
|
||||
/* Only use the specified SSL certificate and private key if the resolved URL names the same
|
||||
authority and uses the same protocol. */
|
||||
if (result.scheme == config->cacheUri.scheme && result.authority == config->cacheUri.authority) {
|
||||
if (const auto & cert = config->tlsCert.get()) {
|
||||
debug("using TLS client certificate %s for '%s'", PathFmt(*cert), request.uri);
|
||||
request.tlsCert = *cert;
|
||||
}
|
||||
|
||||
if (const auto & key = config->tlsKey.get()) {
|
||||
debug("using TLS client key '%s' for '%s'", PathFmt(*key), request.uri);
|
||||
request.tlsKey = *key;
|
||||
}
|
||||
}
|
||||
|
||||
return request;
|
||||
return FileTransferRequest(result);
|
||||
}
|
||||
|
||||
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
|
||||
|
||||
Loaded 100 of 258 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user